From 79698404b3e594216d70e01dc6275c71320c3ad7 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 27 Oct 2026 05:27:10 +0100 Subject: [PATCH 01/33] [modular] derive required inputs and defaults from the ordered block walk Sequential and loop blocks decide each input's `required` flag and default in one ordered walk over their sub-blocks; `required_inputs` is derived from `inputs` as in conditional blocks, and the two overrides are gone. Composites declare their own `InputParam`/`OutputParam`/`ComponentSpec`/`ConfigSpec` for a derived entry with `*_overrides`, and get a FutureWarning until they settle a default two sub-blocks disagree on, a conditional block's per-branch defaults included. Leaf blocks declare what they write back, docstrings regenerated. Fixes #15009. Co-Authored-By: Claude Fable 5.1 --- .ai/references/modular.md | 18 +- .../sequential_pipeline_blocks.md | 1 + .../anima/modular_blocks_anima.py | 8 +- .../modular_pipelines/cosmos/denoise.py | 2 - .../cosmos/modular_blocks_cosmos3.py | 8 +- .../modular_pipelines/echo/denoise.py | 7 +- .../echo/modular_blocks_echo.py | 6 +- .../modular_pipelines/flux/before_denoise.py | 10 + .../flux/modular_blocks_flux.py | 16 +- .../flux/modular_blocks_flux_kontext.py | 12 + .../flux2/modular_blocks_flux2.py | 6 +- .../flux2/modular_blocks_flux2_klein.py | 12 +- .../flux2/modular_blocks_flux2_klein_base.py | 12 +- .../flux3/modular_blocks_flux3.py | 12 +- .../flux3/modular_blocks_flux3_distilled.py | 12 +- .../modular_pipelines/helios/denoise.py | 18 + .../helios/modular_blocks_helios.py | 10 +- .../helios/modular_blocks_helios_pyramid.py | 6 +- ...modular_blocks_helios_pyramid_distilled.py | 16 +- .../modular_blocks_hunyuan_video1_5.py | 14 +- .../modular_pipelines/ideogram4/denoise.py | 2 - .../ideogram4/modular_blocks_ideogram4.py | 4 +- .../modular_pipelines/krea2/before_denoise.py | 14 +- .../modular_pipelines/ltx/before_denoise.py | 4 + .../ltx/modular_blocks_ltx.py | 26 +- .../ltx2/modular_blocks_ltx2.py | 18 +- .../ltx2/modular_blocks_ltx25.py | 8 +- .../minimax_h3/modular_blocks_minimax_h3.py | 10 +- .../modular_pipelines/modular_pipeline.py | 387 +++++++--- .../modular_pipeline_utils.py | 47 ++ .../qwenimage/before_denoise.py | 6 + .../qwenimage/modular_blocks_qwenimage.py | 14 +- .../modular_blocks_qwenimage_edit.py | 10 +- .../qwenimage21/modular_blocks_qwenimage21.py | 8 +- .../modular_blocks_qwenimage21_turbo.py | 8 +- .../modular_blocks_stable_diffusion_3.py | 14 +- .../modular_blocks_stable_diffusion_xl.py | 14 +- .../modular_pipelines/wan/before_denoise.py | 19 + .../wan/modular_blocks_wan22_i2v.py | 2 +- .../wan/modular_blocks_wan_i2v.py | 12 +- .../z_image/before_denoise.py | 14 + .../z_image/modular_blocks_z_image.py | 10 +- .../test_sequential_pipeline_blocks.py | 669 ++++++++++++++++++ 43 files changed, 1270 insertions(+), 256 deletions(-) create mode 100644 tests/modular_pipelines/test_sequential_pipeline_blocks.py diff --git a/.ai/references/modular.md b/.ai/references/modular.md index 8ab88e65cc98..705119969707 100644 --- a/.ai/references/modular.md +++ b/.ai/references/modular.md @@ -282,20 +282,22 @@ if block_state.num_frames is None: block_state.num_frames = 189 ``` -A declared default is part of the block's contract, so the assembled pipeline is aware of it: the generated docstring shows it and `default_call_parameters` reports it. Resolved inside the body instead, the input renders as `*optional*` with no default, and nothing at the pipeline level can report what the block will actually do. Don't worry about branches of a conditional blockset declaring different defaults for the same input — each branch resolves its own at runtime. Resolve inside `__call__` only when the default is *computed* — derived from other inputs or component config (`height = components.default_sample_size * components.vae_scale_factor`). And when several blocks in a sequence share an input, declare the same default on each (or only on the first block that reads it): in a sequence the input is one shared value, so disagreeing declarations are silently resolved first-block-wins. +A declared default is part of the block's contract, so the assembled pipeline is aware of it: the generated docstring shows it and `default_call_parameters` reports it. Resolved inside the body instead, the input renders as `*optional*` with no default, and nothing at the pipeline level can report what the block will actually do. Don't worry about branches of a conditional blockset declaring different defaults for the same input — each branch resolves its own at runtime. Resolve inside `__call__` only when the default is *computed* — derived from other inputs or component config (`height = components.default_sample_size * components.vae_scale_factor`). And when several blocks in a sequence share an input, declare the same default on each (or only on the first block that reads it): in a sequence the input is one shared value, so the composite warns about disagreeing declarations until it declares the input itself in `input_overrides` (the first declaration wins meanwhile). A conditional block's per-branch defaults count as its opinion in that comparison; a declaration without a default has none. -**A composed `SequentialPipelineBlocks` can override `inputs` / `outputs`.** Two uses: narrow `outputs` to what downstream actually consumes, so the docstring shows the step's product instead of every internal intermediate (Wan-Animate-2's core denoise exposes only `segment_frames`); and change one input default per preset by mapping over `super().inputs` (the distilled core denoise turns `num_inference_steps` into `default=10`): +**A composite replaces what it derives from its sub-blocks with `*_overrides`.** `input_overrides`, `intermediate_output_overrides`, `component_overrides` and `config_overrides` are `{name: declaration}` class attributes on `ConditionalPipelineBlocks`, `SequentialPipelineBlocks` and `LoopSequentialPipelineBlocks` (also `from_blocks_dict` arguments). The name must already be in the derived list, and the declaration, an `InputParam` / `OutputParam` / `ComponentSpec` / `ConfigSpec`, takes the derived entry's place whole, so it states everything: default, `required`, description. Use it to settle a default two sub-blocks disagree on (the composite gets a warning until it does), to set a preset's default, or to reword a description from the composite's point of view. `InputParam.template(...)` keeps a common input to one line. ```python -@property -def inputs(self): +class DistilledCoreDenoiseStep(SequentialPipelineBlocks): + block_classes = [...] + block_names = [...] # The distilled checkpoint samples in few steps. - return [ - InputParam.template("num_inference_steps", default=10) if param.name == "num_inference_steps" else param - for param in super().inputs - ] + input_overrides = {"num_inference_steps": InputParam.template("num_inference_steps", default=10)} ``` +A composite reads its sub-blocks' declarations and never writes to them: the `required` flag it reports is its own view ("the caller has to pass this"), the leaf's flag is the runtime check in `get_block_state`, and the two legitimately differ. Copy an `InputParam` before changing it. + +Overriding the `outputs` property by hand is still the way to narrow `outputs` to what downstream actually consumes, so the docstring shows the step's product instead of every internal intermediate (Wan-Animate-2's core denoise exposes only `segment_frames`). + ## ComponentSpec patterns ```python diff --git a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md index 1bd67e17b8bf..9c7a7d5892cc 100644 --- a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md +++ b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md @@ -118,6 +118,7 @@ There are a few properties you should set: - `description`: We recommend adding a description for the assembled block to explain what the combined step does. - `model_name`: This is automatically derived from the sub-blocks but isn't always correct, so you may need to override it. - `outputs`: By default this is the same as `intermediate_outputs`, but you can manually set it to control which values appear in the doc. This is useful for showing only the final outputs instead of all intermediate values. +- `input_overrides`, `intermediate_output_overrides`, `component_overrides`, `config_overrides`: The composite's own declarations replacing entries of the aggregated lists, as `{name: declaration}`. For example `input_overrides = {"strength": InputParam.template("strength", default=0.9999)}` sets the default the whole sequence uses when two sub-blocks declare different ones. The declaration replaces the aggregated entry whole, and the name must already be in the aggregated list. These properties, together with the aggregated `inputs`, `intermediate_outputs`, and `expected_components`, are used to automatically generate the `doc` property. diff --git a/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py b/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py index f17538fd258f..4de17619f67a 100644 --- a/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py +++ b/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py @@ -236,7 +236,7 @@ class AnimaAutoCoreDenoiseStep(AutoPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. @@ -244,7 +244,7 @@ class AnimaAutoCoreDenoiseStep(AutoPipelineBlocks): Strength for img2img/inpainting. generator (`Generator`, *optional*): Torch generator for deterministic generation. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. **denoiser_input_fields (`None`, *optional*): The conditional model inputs for the Anima denoiser. @@ -342,13 +342,13 @@ class AnimaAutoBlocks(SequentialPipelineBlocks): The number of images to generate per prompt. image_latents (`Tensor`, *optional*): image latents used to guide the image generation. Can be generated from vae_encoder step. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. strength (`float`, *optional*, defaults to 0.9): Strength for img2img/inpainting. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. **denoiser_input_fields (`None`, *optional*): The conditional model inputs for the Anima denoiser. diff --git a/src/diffusers/modular_pipelines/cosmos/denoise.py b/src/diffusers/modular_pipelines/cosmos/denoise.py index 2422defe4d46..b193b4eafb54 100644 --- a/src/diffusers/modular_pipelines/cosmos/denoise.py +++ b/src/diffusers/modular_pipelines/cosmos/denoise.py @@ -993,8 +993,6 @@ class Cosmos3TransferDenoiseStep(Cosmos3DenoiseLoopWrapper): Timestep interval [lo, hi] over which text guidance is active (None = always). control_guidance_interval (`tuple`, *optional*): Timestep interval [lo, hi] over which control guidance is active (None = always). - latents (`Tensor`): - Noisy target latents to update. condition_latents (`Tensor`): Clean target latents on the conditioned frames (the autoregressive seed). diff --git a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py index 864e687adad2..e43b48146509 100644 --- a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py +++ b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py @@ -1068,7 +1068,7 @@ class Cosmos3AutoCoreDenoiseStep(ConditionalPipelineBlocks): Torch generator for deterministic generation. fps (`float`, *optional*, defaults to 24.0): Frame rate of the generated video. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. mixed_precision_format (`str`, *optional*): None follows the ModelOpt FP8 checkpoint schedule; 'none' keeps the native quantized forward; 'fp8' is @@ -1099,7 +1099,7 @@ class Cosmos3AutoCoreDenoiseStep(ConditionalPipelineBlocks): Latent-frame indexes fixed by visual conditioning. num_frames (`int`, *optional*): Number of frames to generate. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy vision latents. sound_latents (`Tensor`, *optional*): Pre-generated noisy sound latents. @@ -1280,7 +1280,7 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): Number of frames the first chunk reuses from the input video. generator (`Generator`, *optional*): Torch generator for deterministic generation. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. mixed_precision_format (`str`, *optional*): None follows the ModelOpt FP8 checkpoint schedule; 'none' keeps the native quantized forward; 'fp8' is @@ -1307,7 +1307,7 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): Vision latents encoded from the conditioning image or video. vision_condition_frames (`list`, *optional*): Latent-frame indexes fixed by visual conditioning. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy vision latents. sound_latents (`Tensor`, *optional*): Pre-generated noisy sound latents. diff --git a/src/diffusers/modular_pipelines/echo/denoise.py b/src/diffusers/modular_pipelines/echo/denoise.py index cbe80c0ab609..cba80d9bd5fd 100644 --- a/src/diffusers/modular_pipelines/echo/denoise.py +++ b/src/diffusers/modular_pipelines/echo/denoise.py @@ -359,7 +359,8 @@ class EchoDenoiseLoopStep(LoopSequentialPipelineBlocks): transformer (`LTX2VideoTransformer3DModel`) Inputs: - sigmas (`list | tuple`): + sigmas (`list | tuple`, *optional*, defaults to (1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, + 0.421875, 0.0)): DMD sigma schedule, including the terminal zero. latents (`Tensor`): Pre-generated noisy latents for image generation. @@ -401,10 +402,6 @@ class EchoDenoiseLoopStep(LoopSequentialPipelineBlocks): Frame rate of the generated video. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. - audio_latents (`Tensor`): - Packed target audio tokens. - sigmas (`list | tuple`): - DMD sigma schedule including the terminal zero. generator (`Generator`, *optional*): Torch generator for deterministic generation. first_frame_tokens (`Tensor`, *optional*): diff --git a/src/diffusers/modular_pipelines/echo/modular_blocks_echo.py b/src/diffusers/modular_pipelines/echo/modular_blocks_echo.py index 16c95d3372bd..40786e7d49f1 100644 --- a/src/diffusers/modular_pipelines/echo/modular_blocks_echo.py +++ b/src/diffusers/modular_pipelines/echo/modular_blocks_echo.py @@ -120,7 +120,8 @@ class EchoDenoiseStep(SequentialPipelineBlocks): Optional initial audio noise in VAE form (B, C, L, M). generator (`Generator`, *optional*): Torch generator for deterministic generation. - sigmas (`list | tuple`): + sigmas (`list | tuple`, *optional*, defaults to (1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, + 0.421875, 0.0)): DMD sigma schedule, including the terminal zero. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. @@ -263,7 +264,8 @@ class EchoBlocks(SequentialPipelineBlocks): Optional initial audio noise in VAE form (B, C, L, M). generator (`Generator`, *optional*): Torch generator for deterministic generation. - sigmas (`list | tuple`): + sigmas (`list | tuple`, *optional*, defaults to (1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, + 0.421875, 0.0)): DMD sigma schedule, including the terminal zero. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. diff --git a/src/diffusers/modular_pipelines/flux/before_denoise.py b/src/diffusers/modular_pipelines/flux/before_denoise.py index 8e5a285dbe11..1f9e708b8408 100644 --- a/src/diffusers/modular_pipelines/flux/before_denoise.py +++ b/src/diffusers/modular_pipelines/flux/before_denoise.py @@ -355,6 +355,16 @@ def intermediate_outputs(self) -> list[OutputParam]: OutputParam( "latents", type_hint=torch.Tensor, description="The initial latents to use for the denoising process" ), + OutputParam( + "height", + type_hint=int, + description="The height of the generated image, resolved to the default when not passed", + ), + OutputParam( + "width", + type_hint=int, + description="The width of the generated image, resolved to the default when not passed", + ), ] @staticmethod diff --git a/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py b/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py index a725304c6af6..464d27c1f278 100644 --- a/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py +++ b/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py @@ -162,6 +162,10 @@ class FluxBeforeDenoiseStep(SequentialPipelineBlocks): Outputs: latents (`Tensor`): The initial latents to use for the denoising process + height (`int`): + The height of the generated image, resolved to the default when not passed + width (`int`): + The width of the generated image, resolved to the default when not passed timesteps (`Tensor`): The timesteps to use for inference num_inference_steps (`int`): @@ -227,6 +231,10 @@ class FluxImg2ImgBeforeDenoiseStep(SequentialPipelineBlocks): Outputs: latents (`Tensor`): The initial latents to use for the denoising process + height (`int`): + The height of the generated image, resolved to the default when not passed + width (`int`): + The width of the generated image, resolved to the default when not passed timesteps (`Tensor`): The timesteps to use for inference num_inference_steps (`int`): @@ -268,9 +276,9 @@ class FluxAutoBeforeDenoiseStep(AutoPipelineBlocks): scheduler (`FlowMatchEulerDiscreteScheduler`) Inputs: - height (`int`): + height (`int`, *optional*): TODO: Add description. - width (`int`): + width (`int`, *optional*): TODO: Add description. latents (`Tensor | NoneType`, *optional*): TODO: Add description. @@ -302,6 +310,10 @@ class FluxAutoBeforeDenoiseStep(AutoPipelineBlocks): Outputs: latents (`Tensor`): The initial latents to use for the denoising process + height (`int`): + The height of the generated image, resolved to the default when not passed + width (`int`): + The width of the generated image, resolved to the default when not passed timesteps (`Tensor`): The timesteps to use for inference num_inference_steps (`int`): diff --git a/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py b/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py index f937d2555b1a..0ac08abfdb65 100644 --- a/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py +++ b/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py @@ -153,6 +153,10 @@ class FluxKontextBeforeDenoiseStep(SequentialPipelineBlocks): Outputs: latents (`Tensor`): The initial latents to use for the denoising process + height (`int`): + The height of the generated image, resolved to the default when not passed + width (`int`): + The width of the generated image, resolved to the default when not passed timesteps (`Tensor`): The timesteps to use for inference num_inference_steps (`int`): @@ -220,6 +224,10 @@ class FluxKontextImageConditionedBeforeDenoiseStep(SequentialPipelineBlocks): Outputs: latents (`Tensor`): The initial latents to use for the denoising process + height (`int`): + The height of the generated image, resolved to the default when not passed + width (`int`): + The width of the generated image, resolved to the default when not passed timesteps (`Tensor`): The timesteps to use for inference num_inference_steps (`int`): @@ -291,6 +299,10 @@ class FluxKontextAutoBeforeDenoiseStep(AutoPipelineBlocks): Outputs: latents (`Tensor`): The initial latents to use for the denoising process + height (`int`): + The height of the generated image, resolved to the default when not passed + width (`int`): + The width of the generated image, resolved to the default when not passed timesteps (`Tensor`): The timesteps to use for inference num_inference_steps (`int`): diff --git a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2.py b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2.py index 2bbb7975a983..8053a6540803 100644 --- a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2.py +++ b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2.py @@ -314,11 +314,11 @@ class Flux2AutoBlocks(SequentialPipelineBlocks): TODO: Add description. image_latents (`list`, *optional*): TODO: Add description. - latents (`Tensor | NoneType`): + latents (`Tensor | NoneType`, *optional*): TODO: Add description. - num_inference_steps (`None`): + num_inference_steps (`None`, *optional*, defaults to 50): TODO: Add description. - timesteps (`None`): + timesteps (`None`, *optional*): TODO: Add description. sigmas (`None`, *optional*): TODO: Add description. diff --git a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein.py b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein.py index 689cf808c4ba..3004cd73ed90 100644 --- a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein.py +++ b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein.py @@ -280,13 +280,13 @@ class Flux2KleinAutoCoreDenoiseStep(AutoPipelineBlocks): TODO: Add description. width (`int`, *optional*): TODO: Add description. - latents (`Tensor | NoneType`): + latents (`Tensor | NoneType`, *optional*): TODO: Add description. generator (`None`, *optional*): TODO: Add description. - num_inference_steps (`None`): + num_inference_steps (`None`, *optional*, defaults to 50): TODO: Add description. - timesteps (`None`): + timesteps (`None`, *optional*): TODO: Add description. sigmas (`None`, *optional*): TODO: Add description. @@ -355,11 +355,11 @@ class Flux2KleinAutoBlocks(SequentialPipelineBlocks): TODO: Add description. image_latents (`list`, *optional*): TODO: Add description. - latents (`Tensor | NoneType`): + latents (`Tensor | NoneType`, *optional*): TODO: Add description. - num_inference_steps (`None`): + num_inference_steps (`None`, *optional*, defaults to 50): TODO: Add description. - timesteps (`None`): + timesteps (`None`, *optional*): TODO: Add description. sigmas (`None`, *optional*): TODO: Add description. diff --git a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein_base.py b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein_base.py index f3108bdadeac..c91c51cf84ce 100644 --- a/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein_base.py +++ b/src/diffusers/modular_pipelines/flux2/modular_blocks_flux2_klein_base.py @@ -291,15 +291,15 @@ class Flux2KleinBaseAutoCoreDenoiseStep(AutoPipelineBlocks): TODO: Add description. width (`int`, *optional*): TODO: Add description. - latents (`Tensor | NoneType`): + latents (`Tensor | NoneType`, *optional*): TODO: Add description. generator (`None`, *optional*): TODO: Add description. image_latents (`list`, *optional*): TODO: Add description. - num_inference_steps (`None`): + num_inference_steps (`None`, *optional*, defaults to 50): TODO: Add description. - timesteps (`None`): + timesteps (`None`, *optional*): TODO: Add description. sigmas (`None`, *optional*): TODO: Add description. @@ -367,13 +367,13 @@ class Flux2KleinBaseAutoBlocks(SequentialPipelineBlocks): TODO: Add description. num_images_per_prompt (`None`, *optional*, defaults to 1): TODO: Add description. - latents (`Tensor | NoneType`): + latents (`Tensor | NoneType`, *optional*): TODO: Add description. image_latents (`list`, *optional*): TODO: Add description. - num_inference_steps (`None`): + num_inference_steps (`None`, *optional*, defaults to 50): TODO: Add description. - timesteps (`None`): + timesteps (`None`, *optional*): TODO: Add description. sigmas (`None`, *optional*): TODO: Add description. diff --git a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py index 31dcc62f7b0d..0dc01cb5cbc3 100644 --- a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py +++ b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py @@ -286,13 +286,13 @@ class Flux3AutoCoreDenoiseStep(AutoPipelineBlocks): The width in pixels of the generated image; defaults to the first reference image's own width reference_crop (`list | NoneType`, *optional*): Per reference, the box of the reference inside its padded canvas or None, from the preprocess step - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. - timesteps (`Tensor`): + timesteps (`Tensor`, *optional*): Timesteps for the denoising process. (overrides `num_inference_steps` when provided) sigmas (`list`, *optional*): Custom sigmas for the denoising process. (overrides `num_inference_steps` when provided) @@ -379,13 +379,13 @@ class Flux3AutoBlocks(SequentialPipelineBlocks): The width in pixels of the generated image; defaults to the first reference image's own width reference_crop (`list | NoneType`, *optional*): Per reference, the box of the reference inside its padded canvas or None, from the preprocess step - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. - timesteps (`Tensor`): + timesteps (`Tensor`, *optional*): Timesteps for the denoising process. (overrides `num_inference_steps` when provided) sigmas (`list`, *optional*): Custom sigmas for the denoising process. (overrides `num_inference_steps` when provided) diff --git a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py index d23f894c6df7..92e2495dcd26 100644 --- a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py +++ b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py @@ -206,13 +206,13 @@ class Flux3DistilledAutoCoreDenoiseStep(AutoPipelineBlocks): The width in pixels of the generated image; defaults to the first reference image's own width reference_crop (`list | NoneType`, *optional*): Per reference, the box of the reference inside its padded canvas or None, from the preprocess step - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 16): The number of denoising steps. - timesteps (`Tensor`): + timesteps (`Tensor`, *optional*): Timesteps for the denoising process. (overrides `num_inference_steps` when provided) sigmas (`list`, *optional*): Custom sigmas for the denoising process. (overrides `num_inference_steps` when provided) @@ -297,13 +297,13 @@ class Flux3DistilledAutoBlocks(SequentialPipelineBlocks): The width in pixels of the generated image; defaults to the first reference image's own width reference_crop (`list | NoneType`, *optional*): Per reference, the box of the reference inside its padded canvas or None, from the preprocess step - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 16): The number of denoising steps. - timesteps (`Tensor`): + timesteps (`Tensor`, *optional*): Timesteps for the denoising process. (overrides `num_inference_steps` when provided) sigmas (`list`, *optional*): Custom sigmas for the denoising process. (overrides `num_inference_steps` when provided) diff --git a/src/diffusers/modular_pipelines/helios/denoise.py b/src/diffusers/modular_pipelines/helios/denoise.py index 32a6f0b11b79..8d28c2ac0934 100644 --- a/src/diffusers/modular_pipelines/helios/denoise.py +++ b/src/diffusers/modular_pipelines/helios/denoise.py @@ -1061,6 +1061,15 @@ class HeliosPyramidDistilledChunkDenoiseStep(HeliosChunkLoopWrapper): HeliosChunkUpdateStep, ] block_names = ["history_slice", "noise_gen", "denoise_inner", "update_chunk"] + # the shared noise-gen step declares the non-distilled schedule; the distilled denoiser runs 2 steps per stage + input_overrides = { + "pyramid_num_inference_steps_list": InputParam( + "pyramid_num_inference_steps_list", + default=[2, 2, 2], + type_hint=list, + description="Number of denoising steps per pyramid stage.", + ) + } @property def description(self) -> str: @@ -1080,6 +1089,15 @@ class HeliosPyramidDistilledI2VChunkDenoiseStep(HeliosChunkLoopWrapper): HeliosChunkUpdateStep, ] block_names = ["history_slice", "noise_gen", "denoise_inner", "update_chunk"] + # the shared noise-gen step declares the non-distilled schedule; the distilled denoiser runs 2 steps per stage + input_overrides = { + "pyramid_num_inference_steps_list": InputParam( + "pyramid_num_inference_steps_list", + default=[2, 2, 2], + type_hint=list, + description="Number of denoising steps per pyramid stage.", + ) + } @property def description(self) -> str: diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py index c3d5cb4efc77..eb0e74c6c2e0 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py @@ -269,7 +269,7 @@ class HeliosV2VCoreDenoiseStep(SequentialPipelineBlocks): text embeddings used to guide the image generation. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): negative text embeddings used to guide the image generation. Can be generated from text_encoder step. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. @@ -382,13 +382,13 @@ class HeliosAutoCoreDenoiseStep(ConditionalPipelineBlocks): Torch generator for deterministic generation. num_frames (`int`, *optional*, defaults to 132): Total number of video frames to generate. - history_sizes (`list`): + history_sizes (`list`, *optional*, defaults to [16, 2, 1]): Sizes of long/mid/short history buffers for temporal context. keep_first_frame (`bool`, *optional*, defaults to True): Whether to keep the first frame as a prefix in history. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. - sigmas (`list`): + sigmas (`list`, *optional*): Custom sigmas for the denoising process. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. @@ -496,13 +496,13 @@ class HeliosAutoBlocks(SequentialPipelineBlocks): Maximum sigma for video latent noise. num_frames (`int`, *optional*, defaults to 132): Total number of video frames to generate. - history_sizes (`list`): + history_sizes (`list`, *optional*, defaults to [16, 2, 1]): Sizes of long/mid/short history buffers for temporal context. keep_first_frame (`bool`, *optional*, defaults to True): Whether to keep the first frame as a prefix in history. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. - sigmas (`list`): + sigmas (`list`, *optional*): Custom sigmas for the denoising process. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py index fea11786de21..7ae868f7e8a7 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py @@ -260,7 +260,7 @@ class HeliosPyramidV2VCoreDenoiseStep(SequentialPipelineBlocks): text embeddings used to guide the image generation. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): negative text embeddings used to guide the image generation. Can be generated from text_encoder step. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. @@ -368,7 +368,7 @@ class HeliosPyramidAutoCoreDenoiseStep(ConditionalPipelineBlocks): Torch generator for deterministic generation. num_frames (`int`, *optional*, defaults to 132): Total number of video frames to generate. - history_sizes (`list`): + history_sizes (`list`, *optional*, defaults to [16, 2, 1]): Sizes of long/mid/short history buffers for temporal context. keep_first_frame (`bool`, *optional*, defaults to True): Whether to keep the first frame as a prefix in history. @@ -478,7 +478,7 @@ class HeliosPyramidAutoBlocks(SequentialPipelineBlocks): Maximum sigma for video latent noise. num_frames (`int`, *optional*, defaults to 132): Total number of video frames to generate. - history_sizes (`list`): + history_sizes (`list`, *optional*, defaults to [16, 2, 1]): Sizes of long/mid/short history buffers for temporal context. keep_first_frame (`bool`, *optional*, defaults to True): Whether to keep the first frame as a prefix in history. diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py index 3e0b32f0df7e..3bba29ac4485 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py @@ -120,7 +120,7 @@ class HeliosPyramidDistilledCoreDenoiseStep(SequentialPipelineBlocks): Sizes of long/mid/short history buffers for temporal context. keep_first_frame (`bool`, *optional*, defaults to True): Whether to keep the first frame as a prefix in history. - pyramid_num_inference_steps_list (`list`, *optional*, defaults to [10, 10, 10]): + pyramid_num_inference_steps_list (`list`, *optional*, defaults to [2, 2, 2]): Number of denoising steps per pyramid stage. generator (`Generator`, *optional*): Torch generator for deterministic generation. @@ -193,7 +193,7 @@ class HeliosPyramidDistilledI2VCoreDenoiseStep(SequentialPipelineBlocks): Sizes of long/mid/short history buffers for temporal context. keep_first_frame (`bool`, *optional*, defaults to True): Whether to keep the first frame as a prefix in history. - pyramid_num_inference_steps_list (`list`, *optional*, defaults to [10, 10, 10]): + pyramid_num_inference_steps_list (`list`, *optional*, defaults to [2, 2, 2]): Number of denoising steps per pyramid stage. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. @@ -261,7 +261,7 @@ class HeliosPyramidDistilledV2VCoreDenoiseStep(SequentialPipelineBlocks): text embeddings used to guide the image generation. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): negative text embeddings used to guide the image generation. Can be generated from text_encoder step. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. @@ -283,7 +283,7 @@ class HeliosPyramidDistilledV2VCoreDenoiseStep(SequentialPipelineBlocks): Sizes of long/mid/short history buffers for temporal context. keep_first_frame (`bool`, *optional*, defaults to True): Whether to keep the first frame as a prefix in history. - pyramid_num_inference_steps_list (`list`, *optional*, defaults to [10, 10, 10]): + pyramid_num_inference_steps_list (`list`, *optional*, defaults to [2, 2, 2]): Number of denoising steps per pyramid stage. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. @@ -370,11 +370,11 @@ class HeliosPyramidDistilledAutoCoreDenoiseStep(ConditionalPipelineBlocks): Torch generator for deterministic generation. num_frames (`int`, *optional*, defaults to 132): Total number of video frames to generate. - history_sizes (`list`): + history_sizes (`list`, *optional*, defaults to [16, 2, 1]): Sizes of long/mid/short history buffers for temporal context. keep_first_frame (`bool`, *optional*, defaults to True): Whether to keep the first frame as a prefix in history. - pyramid_num_inference_steps_list (`list`, *optional*, defaults to [10, 10, 10]): + pyramid_num_inference_steps_list (`list`, *optional*, defaults to [2, 2, 2]): Number of denoising steps per pyramid stage. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. @@ -486,11 +486,11 @@ class HeliosPyramidDistilledAutoBlocks(SequentialPipelineBlocks): Maximum sigma for video latent noise. num_frames (`int`, *optional*, defaults to 132): Total number of video frames to generate. - history_sizes (`list`): + history_sizes (`list`, *optional*, defaults to [16, 2, 1]): Sizes of long/mid/short history buffers for temporal context. keep_first_frame (`bool`, *optional*, defaults to True): Whether to keep the first frame as a prefix in history. - pyramid_num_inference_steps_list (`list`, *optional*, defaults to [10, 10, 10]): + pyramid_num_inference_steps_list (`list`, *optional*, defaults to [2, 2, 2]): Number of denoising steps per pyramid stage. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. diff --git a/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py b/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py index bdbdba1ecdd9..4164a8fed51f 100644 --- a/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py +++ b/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py @@ -332,10 +332,10 @@ class HunyuanVideo15AutoCoreDenoiseStep(AutoPipelineBlocks): Inputs: prompt_embeds (`Tensor`): text embeddings used to guide the image generation. Can be generated from text_encoder step. - batch_size (`int`): + batch_size (`int`, *optional*): Number of prompts, the final batch size of model inputs should be batch_size * num_images_per_prompt. Can be generated in input step. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. @@ -345,7 +345,7 @@ class HunyuanVideo15AutoCoreDenoiseStep(AutoPipelineBlocks): The width in pixels of the generated image. num_frames (`int`, *optional*, defaults to 121): Number of video frames to generate. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. @@ -418,16 +418,16 @@ class HunyuanVideo15AutoBlocks(SequentialPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - batch_size (`int`): + batch_size (`int`, *optional*): Number of prompts, the final batch size of model inputs should be batch_size * num_images_per_prompt. Can be generated in input step. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. num_frames (`int`, *optional*, defaults to 121): Number of video frames to generate. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. @@ -504,7 +504,7 @@ class HunyuanVideo15Image2VideoBlocks(SequentialPipelineBlocks): Pre-generated noisy latents for image generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image_latents (`Tensor`): + image_latents (`Tensor`, *optional*): Pre-encoded image latents from the VAE encoder step, used as conditioning for I2V. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. diff --git a/src/diffusers/modular_pipelines/ideogram4/denoise.py b/src/diffusers/modular_pipelines/ideogram4/denoise.py index db1a708c4315..a173c7af8555 100644 --- a/src/diffusers/modular_pipelines/ideogram4/denoise.py +++ b/src/diffusers/modular_pipelines/ideogram4/denoise.py @@ -236,8 +236,6 @@ class Ideogram4DenoiseStep(LoopSequentialPipelineBlocks): Effective batch size. prompt_embeds (`Tensor`): Packed conditional encoder_hidden_states. - position_ids (`Tensor`): - Conditional 3-axis MRoPE position ids. segment_ids (`Tensor`): Conditional block-diagonal segment ids. indicator (`Tensor`): diff --git a/src/diffusers/modular_pipelines/ideogram4/modular_blocks_ideogram4.py b/src/diffusers/modular_pipelines/ideogram4/modular_blocks_ideogram4.py index 0b788fe236be..80a123296495 100644 --- a/src/diffusers/modular_pipelines/ideogram4/modular_blocks_ideogram4.py +++ b/src/diffusers/modular_pipelines/ideogram4/modular_blocks_ideogram4.py @@ -128,9 +128,9 @@ class Ideogram4AutoBlocks(SequentialPipelineBlocks): If True, rewrite the prompt into Ideogram4's native JSON caption before encoding. prompt_upsampling_temperature (`float`, *optional*, defaults to 1.0): Sampling temperature for prompt upsampling. - height (`int`, *optional*): + height (`int`): The height in pixels of the generated image. - width (`int`, *optional*): + width (`int`): The width in pixels of the generated image. max_sequence_length (`int`, *optional*, defaults to 2048): Maximum sequence length for prompt encoding. diff --git a/src/diffusers/modular_pipelines/krea2/before_denoise.py b/src/diffusers/modular_pipelines/krea2/before_denoise.py index 17ed6a3cd376..3e632b981c86 100644 --- a/src/diffusers/modular_pipelines/krea2/before_denoise.py +++ b/src/diffusers/modular_pipelines/krea2/before_denoise.py @@ -386,6 +386,8 @@ class Krea2SetTimestepsStep(ModularPipelineBlocks): Outputs: timesteps (`Tensor`): The denoising timesteps. + num_inference_steps (`int`): + The number of denoising steps. """ model_name = "krea2" @@ -418,7 +420,10 @@ def inputs(self) -> list[InputParam]: @property def intermediate_outputs(self) -> list[OutputParam]: - return [OutputParam(name="timesteps", type_hint=torch.Tensor, description="The denoising timesteps.")] + return [ + OutputParam(name="timesteps", type_hint=torch.Tensor, description="The denoising timesteps."), + OutputParam(name="num_inference_steps", type_hint=int, description="The number of denoising steps."), + ] @torch.no_grad() def __call__( @@ -470,6 +475,8 @@ class Krea2TurboSetTimestepsStep(ModularPipelineBlocks): Outputs: timesteps (`Tensor`): The denoising timesteps. + num_inference_steps (`int`): + The number of denoising steps. """ model_name = "krea2" @@ -496,7 +503,10 @@ def inputs(self) -> list[InputParam]: @property def intermediate_outputs(self) -> list[OutputParam]: - return [OutputParam(name="timesteps", type_hint=torch.Tensor, description="The denoising timesteps.")] + return [ + OutputParam(name="timesteps", type_hint=torch.Tensor, description="The denoising timesteps."), + OutputParam(name="num_inference_steps", type_hint=int, description="The number of denoising steps."), + ] @torch.no_grad() def __call__( diff --git a/src/diffusers/modular_pipelines/ltx/before_denoise.py b/src/diffusers/modular_pipelines/ltx/before_denoise.py index 470ea7325ec7..296895f83d7a 100644 --- a/src/diffusers/modular_pipelines/ltx/before_denoise.py +++ b/src/diffusers/modular_pipelines/ltx/before_denoise.py @@ -128,6 +128,10 @@ def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam("batch_size", type_hint=int), OutputParam("dtype", type_hint=torch.dtype), + OutputParam.template("prompt_embeds"), + OutputParam.template("prompt_embeds_mask", name="prompt_attention_mask"), + OutputParam.template("negative_prompt_embeds"), + OutputParam.template("negative_prompt_embeds_mask", name="negative_prompt_attention_mask"), ] @torch.no_grad() diff --git a/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py b/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py index 0bdaf6ec8816..70cb18bdad29 100644 --- a/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py +++ b/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py @@ -289,25 +289,25 @@ class LTXAutoCoreDenoiseStep(AutoPipelineBlocks): text embeddings used to guide the image generation. Can be generated from text_encoder step. prompt_attention_mask (`Tensor`): mask for the text embeddings. Can be generated from text_encoder step. - negative_prompt_embeds (`Tensor`): + negative_prompt_embeds (`Tensor`, *optional*): negative text embeddings used to guide the image generation. Can be generated from text_encoder step. - negative_prompt_attention_mask (`Tensor`): + negative_prompt_attention_mask (`Tensor`, *optional*): mask for the negative text embeddings. Can be generated from text_encoder step. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. - timesteps (`Tensor`): + timesteps (`Tensor`, *optional*): Timesteps for the denoising process. sigmas (`list`, *optional*): Custom sigmas for the denoising process. - height (`int`): + height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. - width (`int`): + width (`int`, *optional*, defaults to 704): The width in pixels of the generated image. - num_frames (`int`): + num_frames (`int`, *optional*, defaults to 161): TODO: Add description. frame_rate (`int`, *optional*, defaults to 25): TODO: Add description. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. @@ -366,17 +366,17 @@ class LTXAutoBlocks(SequentialPipelineBlocks): Torch generator for deterministic generation. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. - timesteps (`Tensor`): + timesteps (`Tensor`, *optional*): Timesteps for the denoising process. sigmas (`list`, *optional*): Custom sigmas for the denoising process. - num_frames (`int`): + num_frames (`int`, *optional*, defaults to 161): TODO: Add description. frame_rate (`int`, *optional*, defaults to 25): TODO: Add description. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. image_latents (`Tensor`, *optional*): TODO: Add description. @@ -455,7 +455,7 @@ class LTXImage2VideoBlocks(SequentialPipelineBlocks): TODO: Add description. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. - image_latents (`Tensor`): + image_latents (`Tensor`, *optional*): TODO: Add description. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py index aa0990f3cc80..16b176ce5c88 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py @@ -971,7 +971,7 @@ class LTX2AutoCoreDenoiseStep(ConditionalPipelineBlocks): RoPE coordinates for the reference tokens. reference_token_counts (`list`, *optional*): Per-reference token counts, in `reference_conditions` order. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -997,11 +997,11 @@ class LTX2AutoCoreDenoiseStep(ConditionalPipelineBlocks): Torch generator for deterministic generation. reference_cross_mask (`Tensor`, *optional*): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. - timesteps (`Tensor`): + timesteps (`Tensor`, *optional*): Timesteps for the denoising process. - audio_latents (`Tensor`): + audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. dtype (`dtype`): The dtype the model inputs are cast to. @@ -1436,7 +1436,7 @@ class LTX2ImageToVideoBlocks(SequentialPipelineBlocks): noise_scale (`float`, *optional*): Interpolation factor between random noise and any provided latents. `None` (default) resolves to 0.0, which keeps the provided latents. - image_latents (`Tensor`): + image_latents (`Tensor`, *optional*): VAE-encoded reference-image latents used for image-to-video conditioning. audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. @@ -1855,7 +1855,7 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): RoPE coordinates for the reference tokens. reference_token_counts (`list`, *optional*): Per-reference token counts, in `reference_conditions` order. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. noise_scale (`float`, *optional*): Initial noise level for the un-conditioned tokens. `None` (default) resolves to `sigmas[0]` when custom @@ -1864,11 +1864,11 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): Custom sigmas for the denoising process. reference_cross_mask (`Tensor`, *optional*): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. - timesteps (`Tensor`): + timesteps (`Tensor`, *optional*): Timesteps for the denoising process. - audio_latents (`Tensor`): + audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. **denoiser_input_fields (`None`, *optional*): conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py index 3280820eab0c..3b89ddc3bbb7 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py @@ -311,7 +311,7 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): RoPE coordinates for the reference tokens. reference_token_counts (`list`, *optional*): Per-reference token counts, in `reference_conditions` order. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. noise_scale (`float`, *optional*): Initial noise level for the un-conditioned tokens. `None` (default) resolves to `sigmas[0]` when custom @@ -320,11 +320,11 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): Custom sigmas for the denoising process. reference_cross_mask (`Tensor`, *optional*): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. - timesteps (`Tensor`): + timesteps (`Tensor`, *optional*): Timesteps for the denoising process. - audio_latents (`Tensor`): + audio_latents (`Tensor`, *optional*): Optional pre-encoded audio latents; random noise is used when not provided. **denoiser_input_fields (`None`, *optional*): conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. diff --git a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py index e96454a44fa4..318f55dd1506 100644 --- a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py +++ b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py @@ -559,10 +559,10 @@ class MiniMaxH3AutoDenoiseStep(ConditionalPipelineBlocks): generator (`Generator`, *optional*): The generator of the request. The conditioning noise is drawn from it first, one draw per condition, before the noise of the generated rows. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated video noise of shape `(1, 24, num_latent_frames, latent_height, latent_width)`, used instead of the draw. - audio_latents (`Tensor`): + audio_latents (`Tensor`, *optional*): Pre-generated audio noise of shape `(2, 32, num_audio_latents)`. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. @@ -692,7 +692,7 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): Height of the generated video in pixels, a multiple of 32. width (`int`, *optional*): Width of the generated video in pixels, a multiple of 32. - num_frames (`int`, *optional*): + num_frames (`int`, *optional*, defaults to None or 124, depending on the workflow): Number of frames to generate, at the fixed 24 fps. Snapped up to the next `17 * n + 5` the video VAE can decode; the resulting duration must stay between 5 and 15 seconds. To generate a video as long as a reference soundtrack, pass `round(samples / sample_rate * 24)`. @@ -716,10 +716,10 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): generator (`Generator`, *optional*): The generator of the request. The conditioning noise is drawn from it first, one draw per condition, before the noise of the generated rows. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated video noise of shape `(1, 24, num_latent_frames, latent_height, latent_width)`, used instead of the draw. - audio_latents (`Tensor`): + audio_latents (`Tensor`, *optional*): Pre-generated audio noise of shape `(2, 32, num_audio_latents)`. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index e098aca8f0be..f6ee73dff153 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -20,7 +20,7 @@ import warnings from collections import OrderedDict from copy import deepcopy -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any import torch @@ -66,6 +66,7 @@ format_workflow, generate_modular_model_card_content, make_doc_string, + override_attribute, ) @@ -698,12 +699,31 @@ class ConditionalPipelineBlocks(ModularPipelineBlocks): element specifies the trigger input for the corresponding block. default_block_name: Name of the default block to run when no trigger inputs match. If None, this block can be skipped entirely when no trigger inputs are provided. + input_overrides: the composite block's own declarations for inputs it derives from its sub-blocks, as + `{name: InputParam}`, e.g. `{"strength": InputParam.template("strength", default=0.9999)}`. The name must + be a derived input, and the declaration replaces the derived one whole, so it states everything: default, + `required`, description. When the sub-blocks declare different defaults for the same input, each one + keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with + one value. + intermediate_output_overrides: the composite block's own declarations for intermediate outputs it derives + from its sub-blocks, as `{name: OutputParam}`, e.g. + `{"latents": OutputParam("latents", description="Denoised latents.")}`. The name must be a derived output. + component_overrides: the composite block's own specs for components it derives from its sub-blocks, as + `{name: ComponentSpec}`, e.g. `{"text_encoder": ComponentSpec("text_encoder", T5EncoderModel)}`. The name + must be a derived component. + config_overrides: the composite block's own specs for configs it derives from its sub-blocks, as + `{name: ConfigSpec}`, e.g. `{"vae_scale_factor": ConfigSpec("vae_scale_factor", default=16)}`. The name + must be a derived config. """ block_classes = [] block_names = [] block_trigger_inputs = [] default_block_name = None + input_overrides: dict[str, InputParam] = {} + intermediate_output_overrides: dict[str, OutputParam] = {} + component_overrides: dict[str, ComponentSpec] = {} + config_overrides: dict[str, ConfigSpec] = {} def __init__(self): sub_blocks = InsertableDict() @@ -737,7 +757,9 @@ def expected_components(self): for component in block.expected_components: if component not in expected_components: expected_components.append(component) - return expected_components + return override_attribute( + expected_components, self.component_overrides, "component_overrides", self.__class__.__name__ + ) @property def expected_configs(self): @@ -746,7 +768,7 @@ def expected_configs(self): for config in block.expected_configs: if config not in expected_configs: expected_configs.append(config) - return expected_configs + return override_attribute(expected_configs, self.config_overrides, "config_overrides", self.__class__.__name__) @property def required_inputs(self) -> list[str]: @@ -768,19 +790,25 @@ def required_inputs(self) -> list[str]: def inputs(self) -> list[tuple[str, Any]]: named_inputs = [(name, block.inputs) for name, block in self.sub_blocks.items()] combined_inputs = combine_inputs(*named_inputs) - # mark Required inputs only if that input is required by all the blocks - for input_param in combined_inputs: + # required only if every sub-block requires it; + for position, input_param in enumerate(combined_inputs): if input_param.name in self.required_inputs: - input_param.required = True + # `replace` copies so the sub-blocks' declarations stay untouched + combined_inputs[position] = replace(input_param, required=True) else: - input_param.required = False - return combined_inputs + combined_inputs[position] = replace(input_param, required=False) + return override_attribute(combined_inputs, self.input_overrides, "input_overrides", self.__class__.__name__) @property def intermediate_outputs(self) -> list[str]: named_outputs = [(name, block.intermediate_outputs) for name, block in self.sub_blocks.items()] combined_outputs = combine_outputs(*named_outputs) - return combined_outputs + return override_attribute( + combined_outputs, + self.intermediate_output_overrides, + "intermediate_output_overrides", + self.__class__.__name__, + ) @property def outputs(self) -> list[str]: @@ -1051,10 +1079,30 @@ class SequentialPipelineBlocks(ModularPipelineBlocks): Attributes: block_classes: list of block classes to be used block_names: list of prefixes for each block + input_overrides: the composite block's own declarations for inputs it derives from its sub-blocks, as + `{name: InputParam}`, e.g. `{"strength": InputParam.template("strength", default=0.9999)}`. The name must + be a derived input, and the declaration replaces the derived one whole, so it states everything: default, + `required`, description. When the sub-blocks declare different defaults for the same input, each one + keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with + one value. Can also be passed to `from_blocks_dict`. + intermediate_output_overrides: the composite block's own declarations for intermediate outputs it derives + from its sub-blocks, as `{name: OutputParam}`, e.g. + `{"latents": OutputParam("latents", description="Denoised latents.")}`. The name must be a derived output. + Can also be passed to `from_blocks_dict`. + component_overrides: the composite block's own specs for components it derives from its sub-blocks, as + `{name: ComponentSpec}`, e.g. `{"text_encoder": ComponentSpec("text_encoder", T5EncoderModel)}`. The name + must be a derived component. Can also be passed to `from_blocks_dict`. + config_overrides: the composite block's own specs for configs it derives from its sub-blocks, as + `{name: ConfigSpec}`, e.g. `{"vae_scale_factor": ConfigSpec("vae_scale_factor", default=16)}`. The name + must be a derived config. Can also be passed to `from_blocks_dict`. """ block_classes = [] block_names = [] + input_overrides: dict[str, InputParam] = {} + intermediate_output_overrides: dict[str, OutputParam] = {} + component_overrides: dict[str, ComponentSpec] = {} + config_overrides: dict[str, ConfigSpec] = {} @property def description(self): @@ -1071,7 +1119,9 @@ def expected_components(self): for component in block.expected_components: if component not in expected_components: expected_components.append(component) - return expected_components + return override_attribute( + expected_components, self.component_overrides, "component_overrides", self.__class__.__name__ + ) @property def expected_configs(self): @@ -1080,7 +1130,7 @@ def expected_configs(self): for config in block.expected_configs: if config not in expected_configs: expected_configs.append(config) - return expected_configs + return override_attribute(expected_configs, self.config_overrides, "config_overrides", self.__class__.__name__) @property def available_workflows(self): @@ -1114,12 +1164,25 @@ def get_workflow(self, workflow_name: str): @classmethod def from_blocks_dict( - cls, blocks_dict: dict[str, Any], description: str | None = None + cls, + blocks_dict: dict[str, Any], + description: str | None = None, + input_overrides: dict[str, InputParam] | None = None, + intermediate_output_overrides: dict[str, OutputParam] | None = None, + component_overrides: dict[str, ComponentSpec] | None = None, + config_overrides: dict[str, ConfigSpec] | None = None, ) -> "SequentialPipelineBlocks": """Creates a SequentialPipelineBlocks instance from a dictionary of blocks. Args: blocks_dict: Dictionary mapping block names to block classes or instances + description: Description of the new block + input_overrides: the composite's own declarations replacing derived `inputs`, as `{name: InputParam}` + intermediate_output_overrides: the composite's own declarations replacing derived + `intermediate_outputs`, as `{name: OutputParam}` + component_overrides: the composite's own specs replacing derived `expected_components`, as + `{name: ComponentSpec}` + config_overrides: the composite's own specs replacing derived `expected_configs`, as `{name: ConfigSpec}` Returns: A new SequentialPipelineBlocks instance @@ -1140,6 +1203,14 @@ def from_blocks_dict( if description is not None: instance.description = description + for name, value in ( + ("input_overrides", input_overrides), + ("intermediate_output_overrides", intermediate_output_overrides), + ("component_overrides", component_overrides), + ("config_overrides", config_overrides), + ): + if value is not None: + setattr(instance, name, value) return instance @@ -1157,63 +1228,124 @@ def __init__(self): ) def _get_inputs(self): - inputs = [] - outputs = set() - - # Go through all blocks in order - for block in self.sub_blocks.values(): - # Add inputs that aren't in outputs yet + # Walk the sub-blocks in order to derive the pipeline inputs. An input is listed unless an earlier block + # produces it. The first declaration is kept; a later declaration of the same input can only make it required + # or disagree on its default. + # + # The comments below follow one img2img pipeline: + # `text_encoder -> image_encoder -> set_timesteps -> prepare_latents -> denoise`, where `image_encoder` is a + # ConditionalPipelineBlocks without a default block: it runs when `image` is passed and produces + # `image_latents`. + inputs = {} # pipeline inputs + outputs = set() # produced by an earlier block + maybe_outputs = set() # produced by an earlier ConditionalPipelineBlocks, if it runs + for block_name, block in self.sub_blocks.items(): for inp in block.inputs: - if inp.name not in outputs and inp.name not in {input.name for input in inputs}: - inputs.append(inp) - - # Only add outputs if the block cannot be skipped - should_add_outputs = True + # kwargs inputs have no name, key them by "*_" like `combine_inputs` does + key = inp.name if inp.name is not None else f"*_{inp.kwargs_type}" + if key in outputs: + # produced by an earlier block, so not a pipeline input: + # e.g. `denoise` needs `prompt_embeds`, but `text_encoder` produces it + continue + # first declaration, listed as declared: + # e.g. `set_timesteps` declares `num_inference_steps` and no earlier block declared or produced it, so + # it becomes a pipeline input with the default `set_timesteps` gave it + if key not in inputs: + # `replace` copies: the flags below are the composite's view and must not reach the sub-block + inputs[key] = replace(inp) + if inp.required and key in maybe_outputs: + # the block requires it, but an earlier conditional block can produce it, so the pipeline lists + # it optional: + # e.g. `prepare_latents` requires `image_latents`, but `image_encoder` produces it when `image` + # is passed, so `image_latents` is optional for the pipeline: the user can pass `image` instead + inputs[key].required = False + continue + if inp.required: + if inputs[key].default is None and not inputs[key].defaults_by_block and key not in maybe_outputs: + # already listed as optional with no default, nothing earlier can produce it, and now a block + # requires it, so the user has to pass it: + # e.g. `prepare_latents` declares `height` optional, `denoise` requires it, so `height` is a + # required pipeline input. Had `prepare_latents` given `height` a default, or had the branches + # of an earlier conditional block each given it theirs, it would stay optional + inputs[key].required = True + elif not inputs[key].required: + if inputs[key].default is None and not inputs[key].defaults_by_block and inp.defaults_by_block: + # the entry has no default and this conditional block's branches each have their own: carry + # them up, prefixed like `combine_inputs` does, so the pipeline docstring shows them. Not a + # conflict: the branch that runs fills in its own default when it reads the state + # (`get_block_state`): + # e.g. `before_encode` lists `num_frames` with no default and the `denoise` branches default + # it to 124 or require it, so the pipeline says "None or 124, depending on the workflow" + inputs[key].defaults_by_block = { + f"{block_name}.{k}": v for k, v in inp.defaults_by_block.items() + } + elif key not in self.input_overrides: + # the defaults each side has an opinion on: its own, or the per-branch ones a conditional block + # recorded. `None` means no default, not an opinion, so it is left out on both sides + existing, declared = inputs[key], inp + existing_defaults = ( + [existing.default] + if existing.default is not None + else [d for d in (existing.defaults_by_block or {}).values() if d is not None] + ) + declared_defaults = ( + [declared.default] + if declared.default is not None + else [d for d in (declared.defaults_by_block or {}).values() if d is not None] + ) + # (`in` compares with `==`, so list defaults such as `[2, 2, 2]` work too) + if (existing_defaults and any(d not in existing_defaults for d in declared_defaults)) or ( + declared_defaults and any(d not in declared_defaults for d in existing_defaults) + ): + # two optional declarations disagree on the default. In a sequence the input is one shared + # value, so the composite has to pick it with `input_overrides`; until 1.0.0 the first + # declaration wins: + # e.g. `strength` is 0.3 on `set_timesteps` and 0.9999 on `prepare_latents`, so the + # pipeline uses 0.3 and warns until it declares `strength` in `input_overrides` + existing_text = f"default {existing.default!r}" + if existing.defaults_by_block: + existing_text = f"per-branch defaults {existing.defaults_by_block!r}" + declared_text = f"default {declared.default!r}" + if declared.defaults_by_block: + declared_text = f"per-branch defaults {declared.defaults_by_block!r}" + deprecate( + "sub-block default conflict", + "1.0.0", + f"{self.__class__.__name__}: sub-block '{block_name}' declares {declared_text} for " + f"input '{key}' but an earlier sub-block declares {existing_text}. The first " + f"declaration currently wins; this will become an error. Declare the input in " + f"`input_overrides` on {self.__class__.__name__} to pick the default explicitly.", + standard_warn=False, + ) if isinstance(block, ConditionalPipelineBlocks) and block.default_block_name is None: - # ConditionalPipelineBlocks without default can be skipped - should_add_outputs = False - - if should_add_outputs: - # Add this block's outputs - block_intermediate_outputs = [out.name for out in block.intermediate_outputs] - outputs.update(block_intermediate_outputs) + # e.g. `image_encoder` may be skipped, so `image_latents` is only produced if it runs: a "maybe" output + maybe_outputs.update(out.name for out in block.intermediate_outputs) + else: + outputs.update(out.name for out in block.intermediate_outputs) - return inputs + return override_attribute( + list(inputs.values()), self.input_overrides, "input_overrides", self.__class__.__name__ + ) - # YiYi TODO: add test for this @property def inputs(self) -> list[tuple[str, Any]]: return self._get_inputs() - @property - def required_inputs(self) -> list[str]: - # Get the first block from the dictionary - first_block = next(iter(self.sub_blocks.values())) - required_by_any = set(getattr(first_block, "required_inputs", set())) - - # Union with required inputs from all other blocks - for block in list(self.sub_blocks.values())[1:]: - block_required = set(getattr(block, "required_inputs", set())) - required_by_any.update(block_required) - - return list(required_by_any) - @property def intermediate_outputs(self) -> list[str]: - named_outputs = [] - for name, block in self.sub_blocks.items(): - inp_names = {inp.name for inp in block.inputs} - # so we only need to list new variables as intermediate_outputs, but if user wants to list these they modified it's still fine (a.k.a we don't enforce) - # filter out them here so they do not end up as intermediate_outputs - if name not in inp_names: - named_outputs.append((name, block.intermediate_outputs)) + named_outputs = [(name, block.intermediate_outputs) for name, block in self.sub_blocks.items()] combined_outputs = combine_outputs(*named_outputs) - return combined_outputs + return override_attribute( + combined_outputs, + self.intermediate_output_overrides, + "intermediate_output_overrides", + self.__class__.__name__, + ) - # YiYi TODO: I think we can remove the outputs property + # `outputs` is what `doc` and the model card list as the block's outputs. By default that is every intermediate + # output; a composite overrides it to show only what the step is for, e.g. just `latents` for the `denoise` block. @property def outputs(self) -> list[str]: - # return next(reversed(self.sub_blocks.values())).intermediate_outputs return self.intermediate_outputs @torch.no_grad() @@ -1411,11 +1543,31 @@ class LoopSequentialPipelineBlocks(ModularPipelineBlocks): Attributes: block_classes: list of block classes to be used block_names: list of prefixes for each block + input_overrides: the composite block's own declarations for inputs it derives from its sub-blocks, as + `{name: InputParam}`, e.g. `{"strength": InputParam.template("strength", default=0.9999)}`. The name must + be a derived input, and the declaration replaces the derived one whole, so it states everything: default, + `required`, description. When the sub-blocks declare different defaults for the same input, each one + keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with + one value. Can also be passed to `from_blocks_dict`. + intermediate_output_overrides: the composite block's own declarations for intermediate outputs it derives + from its sub-blocks, as `{name: OutputParam}`, e.g. + `{"latents": OutputParam("latents", description="Denoised latents.")}`. The name must be a derived output. + Can also be passed to `from_blocks_dict`. + component_overrides: the composite block's own specs for components it derives from its sub-blocks, as + `{name: ComponentSpec}`, e.g. `{"text_encoder": ComponentSpec("text_encoder", T5EncoderModel)}`. The name + must be a derived component. Can also be passed to `from_blocks_dict`. + config_overrides: the composite block's own specs for configs it derives from its sub-blocks, as + `{name: ConfigSpec}`, e.g. `{"vae_scale_factor": ConfigSpec("vae_scale_factor", default=16)}`. The name + must be a derived config. Can also be passed to `from_blocks_dict`. """ model_name = None block_classes = [] block_names = [] + input_overrides: dict[str, InputParam] = {} + intermediate_output_overrides: dict[str, OutputParam] = {} + component_overrides: dict[str, ComponentSpec] = {} + config_overrides: dict[str, ConfigSpec] = {} @property def description(self) -> str: @@ -1435,14 +1587,6 @@ def loop_inputs(self) -> list[InputParam]: """list of input parameters. Must be implemented by subclasses.""" return [] - @property - def loop_required_inputs(self) -> list[str]: - input_names = [] - for input_param in self.loop_inputs: - if input_param.required: - input_names.append(input_param.name) - return input_names - @property def loop_intermediate_outputs(self) -> list[OutputParam]: """list of intermediate output parameters. Must be implemented by subclasses.""" @@ -1459,7 +1603,9 @@ def expected_components(self): for component in self.loop_expected_components: if component not in expected_components: expected_components.append(component) - return expected_components + return override_attribute( + expected_components, self.component_overrides, "component_overrides", self.__class__.__name__ + ) # modified from SequentialPipelineBlocks to include loop_expected_configs @property @@ -1472,53 +1618,62 @@ def expected_configs(self): for config in self.loop_expected_configs: if config not in expected_configs: expected_configs.append(config) - return expected_configs + return override_attribute(expected_configs, self.config_overrides, "config_overrides", self.__class__.__name__) def _get_inputs(self): - inputs = [] - inputs.extend(self.loop_inputs) - outputs = set() - - for name, block in self.sub_blocks.items(): - # Add inputs that aren't in outputs yet + # Same walk as `SequentialPipelineBlocks._get_inputs`, starting from `loop_inputs`. Loop sub-blocks are leaf + # blocks, so there is no skippable ConditionalPipelineBlocks whose outputs may or may not exist. + inputs = {} + outputs = set() # produced by an earlier block + for inp in self.loop_inputs: + # kwargs inputs have no name, key them by "*_" like `combine_inputs` does + key = inp.name if inp.name is not None else f"*_{inp.kwargs_type}" + inputs[key] = replace(inp) + for block_name, block in self.sub_blocks.items(): for inp in block.inputs: - if inp.name not in outputs and inp not in inputs: - inputs.append(inp) - - # Add this block's outputs - block_intermediate_outputs = [out.name for out in block.intermediate_outputs] - outputs.update(block_intermediate_outputs) - - for input_param in inputs: - if input_param.name in self.required_inputs: - input_param.required = True - else: - input_param.required = False + key = inp.name if inp.name is not None else f"*_{inp.kwargs_type}" + if key in outputs: + continue + if key not in inputs: + # `replace` copies: the flags below are the composite's view and must not reach the sub-block + inputs[key] = replace(inp) + continue + if inp.required: + if inputs[key].default is None: + # nothing earlier defaults or produces it: the caller has to pass it + inputs[key].required = True + elif ( + inp.default is not None + and inputs[key].default is not None + and not inputs[key].required + and inp.default != inputs[key].default + and key not in self.input_overrides + ): + # two optional declarations disagree on the default. In a sequence the input is one shared value, + # so the composite has to pick it with `input_overrides`; until 1.0.0 the first declaration wins. + # `None` means no default, not an opinion. Loop sub-blocks are leaves, so there are no per-branch + # defaults to consider here + deprecate( + "sub-block default conflict", + "1.0.0", + f"{self.__class__.__name__}: sub-block '{block_name}' declares default {inp.default!r} for " + f"input '{key}' but an earlier declaration (a loop input or an earlier sub-block) has " + f"{inputs[key].default!r}. The first declaration currently wins; this will become an error. " + f"Declare the input in `input_overrides` on {self.__class__.__name__} to pick the default " + f"explicitly.", + standard_warn=False, + ) + outputs.update(out.name for out in block.intermediate_outputs) - return inputs + return override_attribute( + list(inputs.values()), self.input_overrides, "input_overrides", self.__class__.__name__ + ) @property # Copied from diffusers.modular_pipelines.modular_pipeline.SequentialPipelineBlocks.inputs def inputs(self): return self._get_inputs() - # modified from SequentialPipelineBlocks, if any additionan input required by the loop is required by the block - @property - def required_inputs(self) -> list[str]: - # Get the first block from the dictionary - first_block = next(iter(self.sub_blocks.values())) - required_by_any = set(getattr(first_block, "required_inputs", set())) - - required_by_loop = set(getattr(self, "loop_required_inputs", set())) - required_by_any.update(required_by_loop) - - # Union with required inputs from all other blocks - for block in list(self.sub_blocks.values())[1:]: - block_required = set(getattr(block, "required_inputs", set())) - required_by_any.update(block_required) - - return list(required_by_any) - # YiYi TODO: this need to be thought about more # modified from SequentialPipelineBlocks to include loop_intermediate_outputs @property @@ -1528,7 +1683,12 @@ def intermediate_outputs(self) -> list[str]: for output in self.loop_intermediate_outputs: if output.name not in {output.name for output in combined_outputs}: combined_outputs.append(output) - return combined_outputs + return override_attribute( + combined_outputs, + self.intermediate_output_overrides, + "intermediate_output_overrides", + self.__class__.__name__, + ) # YiYi TODO: this need to be thought about more @property @@ -1562,12 +1722,25 @@ def __init__(self): ) @classmethod - def from_blocks_dict(cls, blocks_dict: dict[str, Any]) -> "LoopSequentialPipelineBlocks": + def from_blocks_dict( + cls, + blocks_dict: dict[str, Any], + input_overrides: dict[str, InputParam] | None = None, + intermediate_output_overrides: dict[str, OutputParam] | None = None, + component_overrides: dict[str, ComponentSpec] | None = None, + config_overrides: dict[str, ConfigSpec] | None = None, + ) -> "LoopSequentialPipelineBlocks": """ Creates a LoopSequentialPipelineBlocks instance from a dictionary of blocks. Args: blocks_dict: Dictionary mapping block names to block instances + input_overrides: the composite's own declarations replacing derived `inputs`, as `{name: InputParam}` + intermediate_output_overrides: the composite's own declarations replacing derived + `intermediate_outputs`, as `{name: OutputParam}` + component_overrides: the composite's own specs replacing derived `expected_components`, as + `{name: ComponentSpec}` + config_overrides: the composite's own specs replacing derived `expected_configs`, as `{name: ConfigSpec}` Returns: A new LoopSequentialPipelineBlocks instance @@ -1585,6 +1758,14 @@ def from_blocks_dict(cls, blocks_dict: dict[str, Any]) -> "LoopSequentialPipelin instance.block_classes = [block.__class__ for block in blocks_dict.values()] instance.block_names = list(blocks_dict.keys()) instance.sub_blocks = blocks_dict + for name, value in ( + ("input_overrides", input_overrides), + ("intermediate_output_overrides", intermediate_output_overrides), + ("component_overrides", component_overrides), + ("config_overrides", config_overrides), + ): + if value is not None: + setattr(instance, name, value) return instance def loop_step(self, components, state: PipelineState, **kwargs): diff --git a/src/diffusers/modular_pipelines/modular_pipeline_utils.py b/src/diffusers/modular_pipelines/modular_pipeline_utils.py index 09960243e8b2..414e461bfbe4 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline_utils.py +++ b/src/diffusers/modular_pipelines/modular_pipeline_utils.py @@ -1176,6 +1176,53 @@ def combine_outputs(*named_output_lists: list[tuple[str, list[OutputParam]]]) -> return list(combined_dict.values()) +def override_attribute(items: list, overrides: dict[str, Any], attr_name: str, block_name: str) -> list: + """ + Apply a composite block's `*_overrides` to the list it derived from its sub-blocks. + + Args: + items: the derived `InputParam`, `OutputParam`, `ComponentSpec` or `ConfigSpec` list + overrides: `{name: declaration}`; every name must be in `items`, and the declaration, an object of the same + class as the entry it replaces, takes that entry's place whole + attr_name: the overrides attribute being applied, e.g. `"input_overrides"`, for error messages + block_name: the composite's class name, for error messages + + Returns: + `items` with each overridden entry replaced by the composite's own declaration; the sub-blocks' declarations + are not touched. + + Example: + ```python + inputs = [InputParam("strength", default=0.3), InputParam("prompt", required=True)] + override_attribute( + inputs, {"strength": InputParam.template("strength", default=0.9999)}, "input_overrides", "InpaintStep" + ) + # [, ] + ``` + """ + if not overrides: + return items + # kwargs inputs/outputs have no name and are keyed by "*_", like `combine_inputs` does + index = { + item.name if item.name is not None else f"*_{item.kwargs_type}": position + for position, item in enumerate(items) + } + for name, declaration in overrides.items(): + if name not in index: + raise ValueError( + f"{block_name}.{attr_name} has an entry for '{name}', which this block does not derive from its " + f"sub-blocks. Available: {sorted(k for k in index if not k.startswith('*_'))}" + ) + expected = type(items[index[name]]) + if not isinstance(declaration, expected): + raise TypeError( + f"{block_name}.{attr_name}['{name}'] must be of type {expected.__name__}, replacing the derived one; " + f"got {type(declaration).__name__}." + ) + items[index[name]] = declaration + return items + + def generate_modular_model_card_content(blocks) -> dict[str, Any]: """ Generate model card content for a modular pipeline. diff --git a/src/diffusers/modular_pipelines/qwenimage/before_denoise.py b/src/diffusers/modular_pipelines/qwenimage/before_denoise.py index e4720b16244d..f024db511aa2 100644 --- a/src/diffusers/modular_pipelines/qwenimage/before_denoise.py +++ b/src/diffusers/modular_pipelines/qwenimage/before_denoise.py @@ -574,6 +574,8 @@ class QwenImageSetTimestepsStep(ModularPipelineBlocks): Outputs: timesteps (`Tensor`): The timesteps to use for the denoising process + num_inference_steps (`int`): + The number of denoising steps """ model_name = "qwenimage" @@ -607,6 +609,7 @@ def intermediate_outputs(self) -> list[OutputParam]: OutputParam( name="timesteps", type_hint=torch.Tensor, description="The timesteps to use for the denoising process" ), + OutputParam(name="num_inference_steps", type_hint=int, description="The number of denoising steps"), ] def __call__( @@ -662,6 +665,8 @@ class QwenImageLayeredSetTimestepsStep(ModularPipelineBlocks): Outputs: timesteps (`Tensor`): The timesteps to use for the denoising process. + num_inference_steps (`int`): + The number of denoising steps. """ model_name = "qwenimage-layered" @@ -690,6 +695,7 @@ def intermediate_outputs(self) -> list[OutputParam]: OutputParam( name="timesteps", type_hint=torch.Tensor, description="The timesteps to use for the denoising process." ), + OutputParam(name="num_inference_steps", type_hint=int, description="The number of denoising steps."), ] @torch.no_grad() diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py index bf87028b2f90..cd66eefa7201 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py @@ -340,7 +340,7 @@ class QwenImageInpaintInputStep(SequentialPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`): image latents used to guide the image generation. Can be generated from vae_encoder step. processed_mask_image (`Tensor`, *optional*): The processed mask image @@ -543,7 +543,7 @@ class QwenImageInpaintCoreDenoiseStep(SequentialPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`): image latents used to guide the image generation. Can be generated from vae_encoder step. processed_mask_image (`Tensor`, *optional*): The processed mask image @@ -789,7 +789,7 @@ class QwenImageControlNetInpaintCoreDenoiseStep(SequentialPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`): image latents used to guide the image generation. Can be generated from vae_encoder step. processed_mask_image (`Tensor`, *optional*): The processed mask image @@ -1152,17 +1152,17 @@ class QwenImageAutoBlocks(SequentialPipelineBlocks): Control image for ControlNet conditioning. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - prompt_embeds (`Tensor`): + prompt_embeds (`Tensor`, *optional*): text embeddings used to guide the image generation. Can be generated from text_encoder step. - prompt_embeds_mask (`Tensor`): + prompt_embeds_mask (`Tensor`, *optional*): mask for the text embeddings. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): negative text embeddings used to guide the image generation. Can be generated from text_encoder step. negative_prompt_embeds_mask (`Tensor`, *optional*): mask for the negative text embeddings. Can be generated from text_encoder step. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py index 37b80b69ec7e..e5f8753253f8 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py @@ -744,17 +744,17 @@ class QwenImageEditAutoBlocks(SequentialPipelineBlocks): Torch generator for deterministic generation. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - height (`int`): + height (`int`, *optional*): The height in pixels of the generated image. - width (`int`): + width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`Tensor`): + image_latents (`Tensor`, *optional*): image latents used to guide the image generation. Can be generated from vae_encoder step. processed_mask_image (`Tensor`, *optional*): The processed mask image - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. diff --git a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py index 569ffe530186..c4c0301604bd 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py +++ b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py @@ -440,13 +440,13 @@ class QwenImage21AutoCoreDenoiseStep(AutoPipelineBlocks): The width in pixels of the generated image. image_latents (`list`, *optional*): Normalized latents of each condition image. Can be generated from vae_encoder step. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. generator (`Generator`, *optional*): Torch generator for deterministic generation. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 40): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. @@ -528,9 +528,9 @@ class QwenImage21AutoBlocks(SequentialPipelineBlocks): The width in pixels of the generated image. image_latents (`list`, *optional*): Normalized latents of each condition image. Can be generated from vae_encoder step. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 40): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. diff --git a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py index e7ba91f01627..2c975b324cc6 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py +++ b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py @@ -409,13 +409,13 @@ class QwenImage21TurboAutoCoreDenoiseStep(AutoPipelineBlocks): The width in pixels of the generated image. image_latents (`list`, *optional*): Normalized latents of each condition image. Can be generated from vae_encoder step. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. generator (`Generator`, *optional*): Torch generator for deterministic generation. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 8): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. @@ -499,9 +499,9 @@ class QwenImage21TurboAutoBlocks(SequentialPipelineBlocks): The width in pixels of the generated image. image_latents (`list`, *optional*): Normalized latents of each condition image. Can be generated from vae_encoder step. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 8): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. diff --git a/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py b/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py index dc2aae05577f..12d5346387f1 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py @@ -172,7 +172,7 @@ class StableDiffusion3I2ICoreDenoiseStep(SequentialPipelineBlocks): The height in pixels of the generated image. width (`None`, *optional*): The width in pixels of the generated image. - image_latents (`None`, *optional*): + image_latents (`None`): Latent input image_latents to be processed. latents (`Tensor | NoneType`, *optional*): Pre-generated noisy latents to be used as inputs for image generation. @@ -243,13 +243,13 @@ class StableDiffusion3AutoCoreDenoiseStep(AutoPipelineBlocks): The width in pixels of the generated image. image_latents (`None`, *optional*): Latent input image_latents to be processed. - latents (`Tensor | NoneType`): + latents (`Tensor | NoneType`, *optional*): Pre-generated noisy latents to be used as inputs for image generation. generator (`None`, *optional*): One or a list of torch generator(s) to make generation deterministic. - num_inference_steps (`None`): + num_inference_steps (`None`, *optional*, defaults to 50): The number of denoising steps. - timesteps (`None`): + timesteps (`None`, *optional*): Custom timesteps to use for the denoising process. sigmas (`None`, *optional*): Custom sigmas to use for the denoising process. @@ -332,11 +332,11 @@ class StableDiffusion3AutoBlocks(SequentialPipelineBlocks): The number of images to generate per prompt. image_latents (`None`, *optional*): Latent input image_latents to be processed. - latents (`Tensor | NoneType`): + latents (`Tensor | NoneType`, *optional*): Pre-generated noisy latents to be used as inputs for image generation. - num_inference_steps (`None`): + num_inference_steps (`None`, *optional*, defaults to 50): The number of denoising steps. - timesteps (`None`): + timesteps (`None`, *optional*): Custom timesteps to use for the denoising process. sigmas (`None`, *optional*): Custom sigmas to use for the denoising process. diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py index ebe43e4ff23c..45fbeebdb239 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py @@ -14,7 +14,7 @@ from ...utils import logging from ..modular_pipeline import AutoPipelineBlocks, SequentialPipelineBlocks -from ..modular_pipeline_utils import OutputParam +from ..modular_pipeline_utils import InputParam, OutputParam from .before_denoise import ( StableDiffusionXLControlNetInputStep, StableDiffusionXLControlNetUnionInputStep, @@ -127,6 +127,8 @@ class StableDiffusionXLInpaintBeforeDenoiseStep(SequentialPipelineBlocks): StableDiffusionXLImg2ImgPrepareAdditionalConditioningStep, ] block_names = ["set_timesteps", "prepare_latents", "prepare_add_cond"] + # set_timesteps is shared with img2img and declares 0.3; inpainting starts from (almost) pure noise + input_overrides = {"strength": InputParam.template("strength", default=0.9999)} @property def description(self): @@ -363,11 +365,11 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): TODO: Add description. denoising_end (`None`, *optional*): TODO: Add description. - strength (`None`, *optional*, defaults to 0.3): - TODO: Add description. + strength (`float`, *optional*, defaults to 0.9999 or 0.3, depending on the workflow): + Strength for img2img/inpainting. denoising_start (`None`, *optional*): TODO: Add description. - latents (`None`): + latents (`None`, *optional*): TODO: Add description. image_latents (`Tensor`, *optional*): The latents representing the reference image for image-to-image/inpainting generation. Can be generated @@ -422,6 +424,10 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): pooled_prompt_embeds/negative_pooled_prompt_embeds, and ip_adapter_embeds/negative_ip_adapter_embeds (optional).please add `kwargs_type=denoiser_input_fields` to their parameter spec (`OutputParam`) when they are created and added to the pipeline state + **controlnet_kwargs (`None`, *optional*): + additional kwargs for controlnet (e.g. control_type_idx and control_type from the controlnet union input + step )please add `kwargs_type=controlnet_kwargs` to their parameter spec (`OutputParam`) when they are + created and added to the pipeline state eta (`None`, *optional*, defaults to 0.0): TODO: Add description. output_type (`None`, *optional*, defaults to pil): diff --git a/src/diffusers/modular_pipelines/wan/before_denoise.py b/src/diffusers/modular_pipelines/wan/before_denoise.py index 0e7aec5364fb..4d4a61db66d1 100644 --- a/src/diffusers/modular_pipelines/wan/before_denoise.py +++ b/src/diffusers/modular_pipelines/wan/before_denoise.py @@ -233,6 +233,16 @@ def intermediate_outputs(self) -> list[str]: type_hint=torch.dtype, description="Data type of model tensor inputs (determined by `transformer.dtype`)", ), + OutputParam( + "prompt_embeds", + type_hint=torch.Tensor, + description="Text embeddings expanded to batch_size * num_videos_per_prompt", + ), + OutputParam( + "negative_prompt_embeds", + type_hint=torch.Tensor, + description="Negative text embeddings expanded to batch_size * num_videos_per_prompt", + ), ] def check_inputs(self, components, block_state): @@ -474,6 +484,15 @@ def inputs(self) -> list[InputParam]: InputParam("sigmas"), ] + @property + def intermediate_outputs(self) -> list[OutputParam]: + return [ + OutputParam( + "timesteps", type_hint=torch.Tensor, description="The timesteps to use for the denoising process" + ), + OutputParam("num_inference_steps", type_hint=int, description="The number of denoising steps"), + ] + @torch.no_grad() def __call__( self, components: WanModularPipeline, state: PipelineState diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py index 583bcaff1a7c..2570e341f1be 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py @@ -112,7 +112,7 @@ class Wan22Image2VideoCoreDenoiseStep(SequentialPipelineBlocks): TODO: Add description. num_frames (`None`, *optional*): TODO: Add description. - image_condition_latents (`None`, *optional*): + image_condition_latents (`None`): TODO: Add description. num_inference_steps (`None`, *optional*, defaults to 50): TODO: Add description. diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py index cb00e9395169..01bbaef38ea7 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py @@ -340,7 +340,7 @@ class WanImage2VideoCoreDenoiseStep(SequentialPipelineBlocks): TODO: Add description. num_frames (`None`, *optional*): TODO: Add description. - image_condition_latents (`None`, *optional*): + image_condition_latents (`None`): TODO: Add description. num_inference_steps (`None`, *optional*, defaults to 50): TODO: Add description. @@ -362,6 +362,14 @@ class WanImage2VideoCoreDenoiseStep(SequentialPipelineBlocks): Number of prompts, the final batch size of model inputs should be batch_size * num_videos_per_prompt dtype (`dtype`): Data type of model tensor inputs (determined by `transformer.dtype`) + prompt_embeds (`Tensor`): + Text embeddings expanded to batch_size * num_videos_per_prompt + negative_prompt_embeds (`Tensor`): + Negative text embeddings expanded to batch_size * num_videos_per_prompt + timesteps (`Tensor`): + The timesteps to use for the denoising process + num_inference_steps (`int`): + The number of denoising steps latents (`Tensor`): The initial latents to use for the denoising process """ @@ -441,7 +449,7 @@ class WanImage2VideoAutoBlocks(SequentialPipelineBlocks): TODO: Add description. attention_kwargs (`None`, *optional*): TODO: Add description. - image_embeds (`Tensor`): + image_embeds (`Tensor`, *optional*): TODO: Add description. output_type (`str`, *optional*, defaults to np): The output type of the decoded videos diff --git a/src/diffusers/modular_pipelines/z_image/before_denoise.py b/src/diffusers/modular_pipelines/z_image/before_denoise.py index e96d80b4bfca..63fdf7e4bd57 100644 --- a/src/diffusers/modular_pipelines/z_image/before_denoise.py +++ b/src/diffusers/modular_pipelines/z_image/before_denoise.py @@ -527,6 +527,7 @@ def intermediate_outputs(self) -> list[OutputParam]: OutputParam( "timesteps", type_hint=torch.Tensor, description="The timesteps to use for the denoising process" ), + OutputParam("num_inference_steps", type_hint=int, description="The number of denoising steps"), ] @torch.no_grad() @@ -583,6 +584,19 @@ def inputs(self) -> list[InputParam]: InputParam("strength", default=0.6), ] + @property + def intermediate_outputs(self) -> list[OutputParam]: + return [ + OutputParam( + "timesteps", type_hint=torch.Tensor, description="The timesteps to use, trimmed by `strength`" + ), + OutputParam( + "num_inference_steps", + type_hint=int, + description="The number of denoising steps, trimmed by `strength`", + ), + ] + def check_inputs(self, components, block_state): if block_state.strength < 0.0 or block_state.strength > 1.0: raise ValueError(f"Strength must be between 0.0 and 1.0, but got {block_state.strength}") diff --git a/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py b/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py index 1440128a3c55..a0bf2e3a8cc3 100644 --- a/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py +++ b/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py @@ -116,7 +116,7 @@ class ZImageImage2ImageCoreDenoiseStep(SequentialPipelineBlocks): TODO: Add description. width (`None`, *optional*): TODO: Add description. - image_latents (`None`, *optional*): + image_latents (`None`): TODO: Add description. latents (`Tensor | NoneType`, *optional*): TODO: Add description. @@ -190,11 +190,11 @@ class ZImageAutoDenoiseStep(AutoPipelineBlocks): TODO: Add description. image_latents (`None`, *optional*): TODO: Add description. - latents (`Tensor | NoneType`): + latents (`Tensor | NoneType`, *optional*): TODO: Add description. generator (`None`, *optional*): TODO: Add description. - num_inference_steps (`None`): + num_inference_steps (`None`, *optional*, defaults to 8): TODO: Add description. sigmas (`None`, *optional*): TODO: Add description. @@ -295,9 +295,9 @@ class ZImageAutoBlocks(SequentialPipelineBlocks): TODO: Add description. image_latents (`None`, *optional*): TODO: Add description. - latents (`Tensor | NoneType`): + latents (`Tensor | NoneType`, *optional*): TODO: Add description. - num_inference_steps (`None`): + num_inference_steps (`None`, *optional*, defaults to 8): TODO: Add description. sigmas (`None`, *optional*): TODO: Add description. diff --git a/tests/modular_pipelines/test_sequential_pipeline_blocks.py b/tests/modular_pipelines/test_sequential_pipeline_blocks.py new file mode 100644 index 000000000000..3da843d0135c --- /dev/null +++ b/tests/modular_pipelines/test_sequential_pipeline_blocks.py @@ -0,0 +1,669 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings + +import pytest + +from diffusers.modular_pipelines import ( + AutoPipelineBlocks, + ComponentSpec, + ConditionalPipelineBlocks, + ConfigSpec, + InputParam, + LoopSequentialPipelineBlocks, + ModularPipelineBlocks, + OutputParam, + SequentialPipelineBlocks, +) + + +class EncodeImageBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("image", required=True)] + + @property + def intermediate_outputs(self): + return [OutputParam("image_latents")] + + def __call__(self, components, state): + return components, state + + +class PrepareLatentsBlock(ModularPipelineBlocks): + """Takes `latents` as an optional user input and always produces it.""" + + model_name = "test" + + @property + def inputs(self): + return [InputParam("latents"), InputParam("batch_size", required=True)] + + @property + def intermediate_outputs(self): + return [OutputParam("latents")] + + def __call__(self, components, state): + return components, state + + +class DenoiseBlock(ModularPipelineBlocks): + """Requires `latents`, which an earlier block normally produces.""" + + model_name = "test" + + @property + def inputs(self): + return [InputParam("latents", required=True), InputParam("prompt_embeds", required=True)] + + @property + def intermediate_outputs(self): + return [OutputParam("latents")] + + def __call__(self, components, state): + return components, state + + +class SkippableEncodeBlocks(ConditionalPipelineBlocks): + block_classes = [EncodeImageBlock] + block_names = ["encode"] + block_trigger_inputs = ["image"] + default_block_name = None + + def select_block(self, image=None): + return "encode" if image is not None else None + + +def strength_auto_blocks(img2img_default, inpaint_default): + """An auto block whose two branches declare `strength` with their own default (`None` for no default).""" + + class Img2ImgBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("image", required=True), InputParam("strength", default=img2img_default)] + + def __call__(self, components, state): + return components, state + + class InpaintBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("mask", required=True), InputParam("strength", default=inpaint_default)] + + def __call__(self, components, state): + return components, state + + class Auto(AutoPipelineBlocks): + block_classes = [InpaintBlock, Img2ImgBlock] + block_names = ["inpaint", "img2img"] + block_trigger_inputs = ["mask", "image"] + + return Auto + + +def required_flags(blocks, *names): + return {inp.name: inp.required for inp in blocks.inputs if inp.name in names} + + +class TestSequentialPipelineBlocksRequiredInputs: + def test_input_produced_by_earlier_block_is_not_required(self): + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"prepare_latents": PrepareLatentsBlock(), "denoise": DenoiseBlock()} + ) + + assert set(blocks.required_inputs) == {"batch_size", "prompt_embeds"} + assert required_flags(blocks, "latents", "batch_size", "prompt_embeds") == { + "latents": False, + "batch_size": True, + "prompt_embeds": True, + } + + def test_input_consumed_before_it_is_produced_stays_required(self): + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"denoise": DenoiseBlock(), "prepare_latents": PrepareLatentsBlock()} + ) + + assert set(blocks.required_inputs) == {"latents", "batch_size", "prompt_embeds"} + + def test_later_block_requiring_an_optional_input_makes_it_required(self): + class OptionalPromptBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("prompt")] + + def __call__(self, components, state): + return components, state + + class RequiredPromptBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("prompt", required=True)] + + def __call__(self, components, state): + return components, state + + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"optional": OptionalPromptBlock(), "required": RequiredPromptBlock()} + ) + + assert blocks.required_inputs == ["prompt"] + assert required_flags(blocks, "prompt") == {"prompt": True} + + def test_default_from_earlier_block_satisfies_later_requirement(self): + class DefaultStepsBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("num_inference_steps", default=50)] + + def __call__(self, components, state): + return components, state + + class RequiredStepsBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("num_inference_steps", required=True)] + + def __call__(self, components, state): + return components, state + + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"default": DefaultStepsBlock(), "required": RequiredStepsBlock()} + ) + + assert blocks.required_inputs == [] + steps = next(inp for inp in blocks.inputs if inp.name == "num_inference_steps") + assert steps.required is False + assert steps.default == 50 + + def test_composite_flags_never_reach_the_sub_block_declarations(self): + class OptionalBatchBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("batch_size")] + + def __call__(self, components, state): + return components, state + + class NeedsImageLatentsBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("image_latents", required=True)] + + def __call__(self, components, state): + return components, state + + # a later requirement makes the composite's entry required, the first sub-block still says optional + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"optional_batch": OptionalBatchBlock(), "prepare_latents": PrepareLatentsBlock()} + ) + assert required_flags(blocks, "batch_size") == {"batch_size": True} + assert required_flags(blocks.sub_blocks["optional_batch"], "batch_size") == {"batch_size": False} + + # a skippable producer makes the composite's entry optional, the leaf still requires it at runtime + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"encode": SkippableEncodeBlocks(), "needs_latents": NeedsImageLatentsBlock()} + ) + assert required_flags(blocks, "image_latents") == {"image_latents": False} + assert required_flags(blocks.sub_blocks["needs_latents"], "image_latents") == {"image_latents": True} + + # a conditional block only requires what every branch requires, each branch keeps its own flag + class Auto(AutoPipelineBlocks): + block_classes = [NeedsImageLatentsBlock, EncodeImageBlock] + block_names = ["needs_latents", "encode"] + block_trigger_inputs = ["image_latents", None] + + auto = Auto() + assert required_flags(auto, "image_latents", "image") == {"image_latents": False, "image": False} + assert required_flags(auto.sub_blocks["needs_latents"], "image_latents") == {"image_latents": True} + assert required_flags(auto.sub_blocks["encode"], "image") == {"image": True} + + def test_output_of_skippable_block_keeps_required_input_listed_but_optional(self): + class NeedsImageLatentsBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("image_latents", required=True)] + + def __call__(self, components, state): + return components, state + + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"encode": SkippableEncodeBlocks(), "consume": NeedsImageLatentsBlock()} + ) + + # the caller can pass `image` and let the encoder produce `image_latents`, or pass `image_latents` directly, + # so neither is required on its own and both stay listed + assert required_flags(blocks, "image", "image_latents") == {"image": False, "image_latents": False} + assert blocks.required_inputs == [] + + +class TestSequentialPipelineBlocksKwargsInputs: + def test_kwargs_type_inputs_are_kept_apart(self): + class TwoKwargsBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam(kwargs_type="denoiser_input_fields"), InputParam(kwargs_type="controlnet_kwargs")] + + def __call__(self, components, state): + return components, state + + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"first": PrepareLatentsBlock(), "second": TwoKwargsBlock()} + ) + + assert [inp.kwargs_type for inp in blocks.inputs if inp.name is None] == [ + "denoiser_input_fields", + "controlnet_kwargs", + ] + + +class TestAutoPipelineBlocksRequiredInputs: + def test_inputs_produced_inside_each_workflow_are_not_required(self): + class Text2ImageBlocks(SequentialPipelineBlocks): + block_classes = [PrepareLatentsBlock, DenoiseBlock] + block_names = ["prepare_latents", "denoise"] + + class Image2ImageBlocks(SequentialPipelineBlocks): + block_classes = [EncodeImageBlock, PrepareLatentsBlock, DenoiseBlock] + block_names = ["encode", "prepare_latents", "denoise"] + + class AutoBlocks(AutoPipelineBlocks): + block_classes = [Image2ImageBlocks, Text2ImageBlocks] + block_names = ["img2img", "text2img"] + block_trigger_inputs = ["image", None] + + blocks = AutoBlocks() + + assert set(blocks.required_inputs) == {"batch_size", "prompt_embeds"} + assert required_flags(blocks, "latents", "image", "batch_size", "prompt_embeds") == { + "latents": False, + "image": False, + "batch_size": True, + "prompt_embeds": True, + } + + +class TestLoopSequentialPipelineBlocksRequiredInputs: + def test_loop_inputs_and_unsatisfied_step_inputs_are_required(self): + class DenoiserStep(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("latents", required=True), InputParam(kwargs_type="denoiser_input_fields")] + + @property + def intermediate_outputs(self): + return [OutputParam("noise_pred")] + + def __call__(self, components, state, i, t): + return components, state + + class SchedulerStep(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [ + InputParam("noise_pred", required=True), + InputParam("latents", required=True), + InputParam(kwargs_type="scheduler_kwargs"), + ] + + @property + def intermediate_outputs(self): + return [OutputParam("latents")] + + def __call__(self, components, state, i, t): + return components, state + + class DenoiseLoop(LoopSequentialPipelineBlocks): + model_name = "test" + block_classes = [DenoiserStep, SchedulerStep] + block_names = ["denoiser", "scheduler"] + + @property + def loop_inputs(self): + return [InputParam("timesteps", required=True), InputParam("num_inference_steps")] + + blocks = DenoiseLoop() + + assert set(blocks.required_inputs) == {"timesteps", "latents"} + assert "noise_pred" not in [inp.name for inp in blocks.inputs] + assert required_flags(blocks, "timesteps", "num_inference_steps", "latents") == { + "timesteps": True, + "num_inference_steps": False, + "latents": True, + } + assert [inp.kwargs_type for inp in blocks.inputs if inp.name is None] == [ + "denoiser_input_fields", + "scheduler_kwargs", + ] + + def test_later_step_requirement_makes_an_optional_loop_input_required(self): + class Step(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("num_inference_steps", required=True), InputParam("guidance_scale", required=True)] + + def __call__(self, components, state, i, t): + return components, state + + class Loop(LoopSequentialPipelineBlocks): + model_name = "test" + block_classes = [Step] + block_names = ["step"] + + @property + def loop_inputs(self): + return [InputParam("num_inference_steps"), InputParam("guidance_scale", default=7.5)] + + # no default on the loop declaration, so the step's requirement wins; a default satisfies it + assert required_flags(Loop(), "num_inference_steps", "guidance_scale") == { + "num_inference_steps": True, + "guidance_scale": False, + } + + +class TestDefaultConflicts: + def _blocks(self, first_default, second_default): + class FirstBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("strength", default=first_default)] + + def __call__(self, components, state): + return components, state + + class SecondBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("strength", default=second_default)] + + def __call__(self, components, state): + return components, state + + return FirstBlock, SecondBlock + + def test_conflicting_defaults_are_deprecated_first_wins(self): + FirstBlock, SecondBlock = self._blocks(0.3, 0.9999) + + class Blocks(SequentialPipelineBlocks): + block_classes = [FirstBlock, SecondBlock] + block_names = ["first", "second"] + + with pytest.warns(FutureWarning, match="'second' declares default 0.9999 for input 'strength'"): + inputs = Blocks().inputs + # the first declaration wins until the deprecation turns into an error + assert [inp.default for inp in inputs if inp.name == "strength"] == [0.3] + + def test_later_none_default_is_not_a_conflict(self): + FirstBlock, SecondBlock = self._blocks(0.3, None) + + class Blocks(SequentialPipelineBlocks): + block_classes = [FirstBlock, SecondBlock] + block_names = ["first", "second"] + + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + Blocks().inputs + + def test_default_override_settles_the_conflict(self): + FirstBlock, SecondBlock = self._blocks(0.3, 0.9999) + + class Blocks(SequentialPipelineBlocks): + block_classes = [FirstBlock, SecondBlock] + block_names = ["first", "second"] + input_overrides = {"strength": InputParam("strength", default=0.9999)} + + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + inputs = Blocks().inputs + assert [inp.default for inp in inputs if inp.name == "strength"] == [0.9999] + + def test_loop_input_without_default_is_not_a_conflict(self): + FirstBlock, _ = self._blocks(0.9999, None) + + class Loop(LoopSequentialPipelineBlocks): + model_name = "test" + block_classes = [FirstBlock] + block_names = ["first"] + + @property + def loop_inputs(self): + return [InputParam("timesteps", required=True), InputParam("strength")] + + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + inputs = Loop().inputs + # the loop declaration has no opinion, so the step's default is the one the loop reports + assert [inp.default for inp in inputs if inp.name == "strength"] == [None] + + def test_loop_blocks_deprecate_and_accept_the_override(self): + FirstBlock, SecondBlock = self._blocks(0.3, 0.9999) + + class Loop(LoopSequentialPipelineBlocks): + model_name = "test" + block_classes = [FirstBlock, SecondBlock] + block_names = ["first", "second"] + + @property + def loop_inputs(self): + return [InputParam("timesteps", required=True)] + + with pytest.warns(FutureWarning, match="'strength'"): + Loop().inputs + + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + inputs = Loop.from_blocks_dict( + {"first": FirstBlock(), "second": SecondBlock()}, + input_overrides={"strength": InputParam("strength", default=0.9999)}, + ).inputs + assert [inp.default for inp in inputs if inp.name == "strength"] == [0.9999] + + def test_branch_defaults_against_a_value_are_a_conflict(self): + FirstBlock, _ = self._blocks(0.5, None) + Auto = strength_auto_blocks(0.3, 0.9999) + + class ValueThenAuto(SequentialPipelineBlocks): + block_classes = [FirstBlock, Auto] + block_names = ["first", "auto"] + + with pytest.warns(FutureWarning, match="'auto' declares per-branch defaults .* declares default 0.5"): + inputs = ValueThenAuto().inputs + assert [(inp.default, inp.defaults_by_block) for inp in inputs if inp.name == "strength"] == [(0.5, None)] + + class AutoThenValue(SequentialPipelineBlocks): + block_classes = [Auto, FirstBlock] + block_names = ["auto", "first"] + + with pytest.warns(FutureWarning, match="'first' declares default 0.5 .* declares per-branch defaults"): + AutoThenValue().inputs + + def test_branch_without_default_has_no_opinion(self): + # one branch agrees with the earlier value and the other declares no default: nothing to settle + FirstBlock, _ = self._blocks(0.3, None) + + class Blocks(SequentialPipelineBlocks): + block_classes = [FirstBlock, strength_auto_blocks(0.3, None)] + block_names = ["first", "auto"] + + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + Blocks().inputs + + def test_branch_defaults_are_carried_onto_an_entry_without_default(self): + FirstBlock, _ = self._blocks(None, None) + + class Blocks(SequentialPipelineBlocks): + block_classes = [FirstBlock, strength_auto_blocks(0.3, 0.9999)] + block_names = ["first", "auto"] + + blocks = Blocks() + with warnings.catch_warnings(): + warnings.simplefilter("error", FutureWarning) + (strength,) = [inp for inp in blocks.inputs if inp.name == "strength"] + assert strength.default is None + assert strength.defaults_by_block == {"auto.inpaint": 0.9999, "auto.img2img": 0.3} + (first,) = [inp for inp in blocks.sub_blocks["first"].inputs if inp.name == "strength"] + assert first.defaults_by_block is None + + def test_branch_defaults_satisfy_a_later_requirement(self): + class RequiresStrengthBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("strength", required=True)] + + def __call__(self, components, state): + return components, state + + class Blocks(SequentialPipelineBlocks): + block_classes = [strength_auto_blocks(0.3, 0.9999), RequiresStrengthBlock] + block_names = ["auto", "requires"] + + # the branch that runs fills in its default before `requires` reads it + assert required_flags(Blocks(), "strength") == {"strength": False} + + +class UNetBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def expected_components(self): + return [ComponentSpec("unet", description="the denoiser")] + + @property + def expected_configs(self): + return [ConfigSpec("scale", default=8)] + + @property + def inputs(self): + return [InputParam("latents", description="noisy latents"), InputParam("batch_size", required=True)] + + @property + def intermediate_outputs(self): + return [OutputParam("latents", description="denoised latents")] + + def __call__(self, components, state): + return components, state + + +class TestOverrides: + def _sequential(self, **overrides): + class Blocks(SequentialPipelineBlocks): + block_classes = [PrepareLatentsBlock, UNetBlock] + block_names = ["prepare_latents", "unet"] + + for name, value in overrides.items(): + setattr(Blocks, name, value) + return Blocks() + + def test_input_override_replaces_the_derived_declaration(self): + blocks = self._sequential(input_overrides={"batch_size": InputParam("batch_size", default=1)}) + (batch_size,) = [inp for inp in blocks.inputs if inp.name == "batch_size"] + assert batch_size.default == 1 + assert batch_size.required is False + assert "batch_size" not in blocks.required_inputs + + def test_override_does_not_mutate_the_sub_block_declaration(self): + blocks = self._sequential(input_overrides={"batch_size": InputParam("batch_size", default=1)}) + _ = blocks.inputs + (batch_size,) = [inp for inp in blocks.sub_blocks["prepare_latents"].inputs if inp.name == "batch_size"] + assert batch_size.required is True + assert batch_size.default is None + + def test_output_component_and_config_overrides(self): + blocks = self._sequential( + intermediate_output_overrides={"latents": OutputParam("latents", description="final latents")}, + component_overrides={"unet": ComponentSpec("unet", description="any UNet2DConditionModel")}, + config_overrides={"scale": ConfigSpec("scale", default=16)}, + ) + assert [out.description for out in blocks.intermediate_outputs if out.name == "latents"] == ["final latents"] + assert [c.description for c in blocks.expected_components if c.name == "unet"] == ["any UNet2DConditionModel"] + assert [c.default for c in blocks.expected_configs if c.name == "scale"] == [16] + + def test_unknown_name_raises(self): + blocks = self._sequential(input_overrides={"strenght": InputParam("strenght", default=0.5)}) + with pytest.raises(ValueError, match="strenght"): + blocks.inputs + blocks = self._sequential(component_overrides={"vae": ComponentSpec("vae", description="x")}) + with pytest.raises(ValueError, match="vae"): + blocks.expected_components + + def test_partial_dict_override_raises(self): + blocks = self._sequential(input_overrides={"batch_size": {"default": 1}}) + with pytest.raises(TypeError, match="must be of type InputParam"): + blocks.inputs + + def test_from_blocks_dict_accepts_overrides(self): + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"prepare_latents": PrepareLatentsBlock(), "unet": UNetBlock()}, + input_overrides={"batch_size": InputParam("batch_size", default=2)}, + intermediate_output_overrides={"latents": OutputParam("latents", description="final latents")}, + config_overrides={"scale": ConfigSpec("scale", default=16)}, + ) + assert [inp.default for inp in blocks.inputs if inp.name == "batch_size"] == [2] + assert [out.description for out in blocks.intermediate_outputs if out.name == "latents"] == ["final latents"] + assert [c.default for c in blocks.expected_configs if c.name == "scale"] == [16] + + loop = LoopSequentialPipelineBlocks.from_blocks_dict( + {"unet": UNetBlock()}, + component_overrides={"unet": ComponentSpec("unet", description="any UNet2DConditionModel")}, + ) + assert [c.description for c in loop.expected_components if c.name == "unet"] == ["any UNet2DConditionModel"] + + def test_auto_blocks_override_replaces_the_per_block_defaults(self): + Auto = strength_auto_blocks(0.3, 0.9999) + + (strength,) = [inp for inp in Auto().inputs if inp.name == "strength"] + assert strength.default is None + assert strength.defaults_by_block == {"inpaint": 0.9999, "img2img": 0.3} + + class AutoWithDefault(Auto): + input_overrides = {"strength": InputParam("strength", default=0.5)} + + (strength,) = [inp for inp in AutoWithDefault().inputs if inp.name == "strength"] + assert strength.default == 0.5 + assert strength.defaults_by_block is None From 7ab5f5fc30205dddaad92039d76ad3abe16f1ad6 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Fri, 6 Nov 2026 23:17:31 +0100 Subject: [PATCH 02/33] [modular] a sequence keeps the first default and warns on a later different one In a sequence an input is one shared value, so the walk keeps the first declaration and warns when a later sub-block declares a different default, until the composite declares the input in `input_overrides`. A later `None` or a conditional sub-block's per-branch defaults are fallbacks and never warn; an earlier default satisfies a later requirement. An output only some branches of a conditional sub-block produce keeps a later requirement listed, optional. The conditional merge is rewritten around `required` / `same_default` and sets `defaults_by_block` through `replace`; the field stays a plain public field. Docstrings read "required or defaults to 124, depending on the workflow". Leaf declarations fixed along the way: the LTX denoise loop steps and the unpack step of one unreleased pipeline mark `height`/`width`/`num_frames` required, `WanImageResizeStep` declares the `height`/`width` it computes as outputs, the Wan 2.2 I2V, Wan 2.2 VACE, Wan I2V auto and LTX 2.5 auto blocks declare `num_frames` (and `height`/`width`) in `input_overrides`, and `MiniMaxH3Blocks` declares `num_frames` with no default. Class docstrings regenerated in the files touched; guides updated. Co-Authored-By: Claude Fable 5.1 --- .ai/references/modular.md | 42 ++- .../sequential_pipeline_blocks.md | 21 +- .../cosmos/modular_blocks_cosmos3.py | 52 ++-- .../flux/modular_blocks_flux_kontext.py | 8 + .../modular_pipelines/flux3/decoders.py | 4 +- .../modular_pipelines/ltx/denoise.py | 18 +- .../ltx2/modular_blocks_ltx2.py | 80 +++--- .../ltx2/modular_blocks_ltx25.py | 37 ++- .../minimax_h3/modular_blocks_minimax_h3.py | 47 ++-- .../modular_pipelines/modular_pipeline.py | 262 ++++++++++-------- .../modular_pipeline_utils.py | 119 +++----- .../modular_blocks_stable_diffusion_xl.py | 20 +- .../modular_pipelines/wan/encoders.py | 3 + .../wan/modular_blocks_wan22_i2v.py | 15 +- .../wan/modular_blocks_wan_i2v.py | 55 +++- .../wan/modular_blocks_wan_vace.py | 11 +- .../test_conditional_pipeline_blocks.py | 137 ++++++--- .../test_sequential_pipeline_blocks.py | 230 +++++++++++---- 18 files changed, 745 insertions(+), 416 deletions(-) diff --git a/.ai/references/modular.md b/.ai/references/modular.md index 705119969707..e298997915bd 100644 --- a/.ai/references/modular.md +++ b/.ai/references/modular.md @@ -282,9 +282,47 @@ if block_state.num_frames is None: block_state.num_frames = 189 ``` -A declared default is part of the block's contract, so the assembled pipeline is aware of it: the generated docstring shows it and `default_call_parameters` reports it. Resolved inside the body instead, the input renders as `*optional*` with no default, and nothing at the pipeline level can report what the block will actually do. Don't worry about branches of a conditional blockset declaring different defaults for the same input — each branch resolves its own at runtime. Resolve inside `__call__` only when the default is *computed* — derived from other inputs or component config (`height = components.default_sample_size * components.vae_scale_factor`). And when several blocks in a sequence share an input, declare the same default on each (or only on the first block that reads it): in a sequence the input is one shared value, so the composite warns about disagreeing declarations until it declares the input itself in `input_overrides` (the first declaration wins meanwhile). A conditional block's per-branch defaults count as its opinion in that comparison; a declaration without a default has none. +A declared default is part of the block's contract, so the assembled pipeline is aware of it. The generated docstring shows it: -**A composite replaces what it derives from its sub-blocks with `*_overrides`.** `input_overrides`, `intermediate_output_overrides`, `component_overrides` and `config_overrides` are `{name: declaration}` class attributes on `ConditionalPipelineBlocks`, `SequentialPipelineBlocks` and `LoopSequentialPipelineBlocks` (also `from_blocks_dict` arguments). The name must already be in the derived list, and the declaration, an `InputParam` / `OutputParam` / `ComponentSpec` / `ConfigSpec`, takes the derived entry's place whole, so it states everything: default, `required`, description. Use it to settle a default two sub-blocks disagree on (the composite gets a warning until it does), to set a preset's default, or to reword a description from the composite's point of view. `InputParam.template(...)` keeps a common input to one line. +``` +num_frames (`int`, *optional*, defaults to 189): + Number of frames to generate. +``` + +and `default_call_parameters` reports it. If you resolve it inside the body instead, the input renders as `*optional*` with no default, and nothing at the pipeline level can report what the block will actually do. Resolve inside `__call__` only when the default is *computed*, derived from other inputs or component config: + +```python +if block_state.height is None: + block_state.height = components.default_sample_size * components.vae_scale_factor +``` + +The other two ways to declare an input: +(1) `required=True` when the block cannot work without the value, so the caller has to pass it; the block raises `ValueError("Required input 'x' is missing")` when it is not passed or `None`. +(2) Optional with no default when the block works with `None`. Most often that is the computed case above, and the block should then also declare the value as an output. Sometimes the block only uses the value to decide something else: e.g. in the text encoder blocks, `negative_prompt` defaults to `None` instead of `""`, so the encoder can tell "none given" from an empty string. It encodes an empty string in its place only when guidance needs one, and ignores a given negative prompt when guidance is disabled (some encoders log a warning). + +Inputs are declared on leaf blocks. When blocks are assembled, the composite derives its inputs from them, and the same input is sometimes declared by more than one sub-block. + +In a conditional block, the sub-blocks may declare different defaults for the same input, and each is respected: e.g. for `strength`, the img2img branch may defaults to 0.3 and the inpaint branch to 0.9999, the docstring of the conditional block says "defaults to 0.3 or 0.9999, depending on the workflow", and at runtime the branch that runs fills in its own. + +In a sequential block, the first sub-block that declares a default sets it for every sub-block after it. When a later sub-block declares a different default, the sequence keeps the first one and warns that the default is ambiguous. We recommend to state it explicitly for such case: declare the input in `input_overrides` on the composite, see below. + +**Redefine what the composite derives with `*_overrides`.** When you assemble blocks, the composite derives its inputs, outputs, components and configs from its sub-blocks. You cannot add or remove any of them there, they are part of the blocks' logic, but you can redefine how one is declared: e.g. you can change the default and description of an input, the description of an output, the repo and loading info of a component, the value of a config. Set `input_overrides`, `intermediate_output_overrides`, `component_overrides` or `config_overrides` as a class attribute on the composite (`ConditionalPipelineBlocks`, `SequentialPipelineBlocks`, `LoopSequentialPipelineBlocks`), or pass them to `from_blocks_dict`: + +```python +# as a class attribute +class MyBlocks(SequentialPipelineBlocks): + block_classes = [TextEncoderStep, DenoiseStep] + block_names = ["text_encoder", "denoise"] + input_overrides = {"num_inference_steps": InputParam.template("num_inference_steps", default=10)} + +# or when assembling from a dict +blocks = SequentialPipelineBlocks.from_blocks_dict( + {"text_encoder": TextEncoderStep(), "denoise": DenoiseStep()}, + input_overrides={"num_inference_steps": InputParam.template("num_inference_steps", default=10)}, +) +``` + +Each is a dict of name to declaration. The name must be one the composite already derives, and the declaration, an `InputParam` / `OutputParam` / `ComponentSpec` / `ConfigSpec`, replaces the derived one whole, so it states everything: default, `required`, description. Typical uses: settle a default two sub-blocks disagree on, set a preset's default as in the distilled example below, or reword a description from the composite's point of view. The per-branch defaults of a conditional block cannot be changed one branch at a time from the composite; for that, change the sub-block. An override on the conditional block replaces them with one default for every branch. `InputParam.template(...)` keeps a common input to one line. ```python class DistilledCoreDenoiseStep(SequentialPipelineBlocks): diff --git a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md index 9c7a7d5892cc..d5d3d53e0412 100644 --- a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md +++ b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md @@ -118,10 +118,29 @@ There are a few properties you should set: - `description`: We recommend adding a description for the assembled block to explain what the combined step does. - `model_name`: This is automatically derived from the sub-blocks but isn't always correct, so you may need to override it. - `outputs`: By default this is the same as `intermediate_outputs`, but you can manually set it to control which values appear in the doc. This is useful for showing only the final outputs instead of all intermediate values. -- `input_overrides`, `intermediate_output_overrides`, `component_overrides`, `config_overrides`: The composite's own declarations replacing entries of the aggregated lists, as `{name: declaration}`. For example `input_overrides = {"strength": InputParam.template("strength", default=0.9999)}` sets the default the whole sequence uses when two sub-blocks declare different ones. The declaration replaces the aggregated entry whole, and the name must already be in the aggregated list. These properties, together with the aggregated `inputs`, `intermediate_outputs`, and `expected_components`, are used to automatically generate the `doc` property. +The aggregated declarations come from the sub-blocks, and the sequence cannot add or remove any of them. It can redefine how one is declared, though, with `input_overrides`, `intermediate_output_overrides`, `component_overrides` and `config_overrides`. You can reuse the same sub-blocks with another default for an input, another type hint or description, or other repo info for a component. Each is a `{name: declaration}` dict. The name must already be in the aggregated list, and the declaration replaces the aggregated entry whole. + +```py +class ImageProcessingStep(SequentialPipelineBlocks): + """ + # auto_docstring + """ + model_name = "my_model" + block_classes = [InputBlock(), ImageEncoderBlock()] + block_names = ["input", "image_encoder"] + # `InputBlock` declares `num_images_per_prompt` without a default; this step defaults it to 1 + input_overrides = { + "num_images_per_prompt": InputParam( + name="num_images_per_prompt", type_hint=int, default=1, description="number of images per prompt" + ) + } +``` + +When two sub-blocks declare different defaults for the same input, the sequence keeps the first one and warns; an `input_overrides` entry explicitly states the default the whole sequence uses. When the input belongs to a conditional sub-block whose branches declare different defaults, the override gives the sequence one value for every workflow instead of the per-branch ones. + Print the `ImageProcessingStep` block to inspect its sub-blocks, and use `doc` for a full summary of the block's inputs, outputs, and components. diff --git a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py index e43b48146509..b1861c86c96c 100644 --- a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py +++ b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py @@ -127,7 +127,7 @@ class Cosmos3AutoTextEncoderStep(AutoPipelineBlocks): default_use_system_prompt (default: True) enable_safety_checker (default: True) Inputs: - control_videos (`dict`, *optional*): + control_videos (`dict`, *optional*, required, depending on the workflow): Mapping of hint name (edge/blur/depth/seg/wsm) to the control video for that modality. height (`int`, *optional*): Height of the generated video in pixels. @@ -145,7 +145,7 @@ class Cosmos3AutoTextEncoderStep(AutoPipelineBlocks): The negative text prompt used for classifier-free guidance. use_system_prompt (`bool`, *optional*, defaults to True or None, depending on the workflow): Whether to prepend the Cosmos3 transfer system prompt. - action (`CosmosActionCondition`, *optional*): + action (`CosmosActionCondition`, *optional*, required, depending on the workflow): Action-conditioning metadata and its reference visual input. fps (`float`, *optional*, defaults to 24.0): Frame rate of the generated video. @@ -207,7 +207,7 @@ class Cosmos3AutoVaeEncoderStep(ConditionalPipelineBlocks): vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) Inputs: - action (`CosmosActionCondition`, *optional*): + action (`CosmosActionCondition`, *optional*, required, depending on the workflow): Action-conditioning metadata and its reference visual input. video (`None`, *optional*): Reference video for video-to-video conditioning. @@ -215,11 +215,11 @@ class Cosmos3AutoVaeEncoderStep(ConditionalPipelineBlocks): Latent-frame indexes to preserve from the conditioning video. condition_video_keep (`str`, *optional*, defaults to first): Which end of a longer conditioning video to use: `first` or `last`. - num_frames (`int`, *optional*): + num_frames (`int`, *optional*, required, depending on the workflow): Number of frames to generate. - height (`int`, *optional*): + height (`int`, *optional*, required, depending on the workflow): Height of the generated video in pixels. - width (`int`, *optional*): + width (`int`, *optional*, required, depending on the workflow): Width of the generated video in pixels. image (`None`, *optional*): Reference image for image-to-video conditioning. @@ -283,7 +283,7 @@ class Cosmos3AutoSoundDecodeStep(AutoPipelineBlocks): sound_tokenizer (`Cosmos3AVAEAudioTokenizer`) Inputs: - sound_latents (`Tensor`, *optional*): + sound_latents (`Tensor`, *optional*, required, depending on the workflow): Denoised sound latents to decode. Outputs: @@ -320,7 +320,7 @@ class Cosmos3DecodeStep(SequentialPipelineBlocks): Denoised vision latents to decode. output_type (`str`, *optional*, defaults to pil): Output format: 'pil', 'np', 'pt'. - sound_latents (`Tensor`, *optional*): + sound_latents (`Tensor`, *optional*, required, depending on the workflow): Denoised sound latents to decode. Outputs: @@ -1046,13 +1046,13 @@ class Cosmos3AutoCoreDenoiseStep(ConditionalPipelineBlocks): Index of the current chunk. previous_output (`None`, *optional*): Decoded pixels of the previous chunk, used to seed later chunks. - control_frames (`dict`, *optional*): + control_frames (`dict`, *optional*, required, depending on the workflow): Preprocessed, time-padded control maps in canonical hint order. - chunk_frames (`int`, *optional*): + chunk_frames (`int`, *optional*, required, depending on the workflow): Pixel frames per chunk. - total_frames (`int`, *optional*): + total_frames (`int`, *optional*, required, depending on the workflow): Total number of output frames. - stride (`int`, *optional*): + stride (`int`, *optional*, required, depending on the workflow): Frame stride between chunks. height (`int`): Height of the generated video in pixels. @@ -1089,21 +1089,21 @@ class Cosmos3AutoCoreDenoiseStep(ConditionalPipelineBlocks): Timestep interval [lo, hi] over which text guidance is active (None = always). control_guidance_interval (`tuple`, *optional*): Timestep interval [lo, hi] over which control guidance is active (None = always). - output_chunks (`list`, *optional*): + output_chunks (`list`, *optional*, required, depending on the workflow): Decoded pixel chunks accumulated so far. - num_chunks (`int`, *optional*): + num_chunks (`int`, *optional*, required, depending on the workflow): Number of autoregressive chunks. x0_tokens_vision (`Tensor`, *optional*): Vision latents encoded from the conditioning image or video. vision_condition_frames (`list`, *optional*): Latent-frame indexes fixed by visual conditioning. - num_frames (`int`, *optional*): + num_frames (`int`, *optional*, required, depending on the workflow): Number of frames to generate. latents (`Tensor`, *optional*): Pre-generated noisy vision latents. sound_latents (`Tensor`, *optional*): Pre-generated noisy sound latents. - action (`CosmosActionCondition`, *optional*): + action (`CosmosActionCondition`, *optional*, required, depending on the workflow): Action-conditioning metadata. action_condition_frame_indexes (`list`, *optional*): Action-frame indexes fixed by action conditioning. @@ -1238,7 +1238,7 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): (default: False) Inputs: - control_videos (`dict`, *optional*): + control_videos (`dict`, *optional*, required, depending on the workflow): Mapping of hint name (edge/blur/depth/seg/wsm) to the control video for that modality. height (`int`, *optional*): Height of the generated video in pixels. @@ -1256,7 +1256,7 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): The negative text prompt used for classifier-free guidance. use_system_prompt (`bool`, *optional*, defaults to True or None, depending on the workflow): Whether to prepend the Cosmos3 transfer system prompt. - action (`CosmosActionCondition`, *optional*): + action (`CosmosActionCondition`, *optional*, required, depending on the workflow): Action-conditioning metadata and its reference visual input. fps (`float`, *optional*, defaults to 24.0): Frame rate of the generated video. @@ -1276,6 +1276,14 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): Index of the current chunk. previous_output (`None`, *optional*): Decoded pixels of the previous chunk, used to seed later chunks. + control_frames (`dict`, *optional*, required, depending on the workflow): + Preprocessed, time-padded control maps in canonical hint order. + chunk_frames (`int`, *optional*, required, depending on the workflow): + Pixel frames per chunk. + total_frames (`int`, *optional*, required, depending on the workflow): + Total number of output frames. + stride (`int`, *optional*, required, depending on the workflow): + Frame stride between chunks. num_first_chunk_conditional_frames (`int`, *optional*, defaults to 0): Number of frames the first chunk reuses from the input video. generator (`Generator`, *optional*): @@ -1301,8 +1309,10 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): Timestep interval [lo, hi] over which text guidance is active (None = always). control_guidance_interval (`tuple`, *optional*): Timestep interval [lo, hi] over which control guidance is active (None = always). - output_chunks (`list`, *optional*): + output_chunks (`list`, *optional*, required, depending on the workflow): Decoded pixel chunks accumulated so far. + num_chunks (`int`, *optional*, required, depending on the workflow): + Number of autoregressive chunks. x0_tokens_vision (`Tensor`, *optional*): Vision latents encoded from the conditioning image or video. vision_condition_frames (`list`, *optional*): @@ -1319,6 +1329,10 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): Whether to generate a synchronized sound track. output_type (`str`, *optional*, defaults to pil): Output format: 'pil', 'np', 'pt'. + action_mode (`str`, *optional*): + Requested action-generation mode. + raw_action_dim_resolved (`int`, *optional*): + Unpadded action-vector dimension. Outputs: videos (`list`): diff --git a/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py b/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py index 0ac08abfdb65..ec8050d26147 100644 --- a/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py +++ b/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py @@ -484,6 +484,10 @@ class FluxKontextCoreDenoiseStep(SequentialPipelineBlocks): TODO: Add description. guidance_scale (`None`, *optional*, defaults to 3.5): TODO: Add description. + image_height (`None`, *optional*): + TODO: Add description. + image_width (`None`, *optional*): + TODO: Add description. joint_attention_kwargs (`None`, *optional*): TODO: Add description. @@ -576,6 +580,10 @@ class FluxKontextAutoBlocks(SequentialPipelineBlocks): TODO: Add description. guidance_scale (`None`, *optional*, defaults to 3.5): TODO: Add description. + image_height (`None`, *optional*): + TODO: Add description. + image_width (`None`, *optional*): + TODO: Add description. output_type (`None`, *optional*, defaults to pil): TODO: Add description. diff --git a/src/diffusers/modular_pipelines/flux3/decoders.py b/src/diffusers/modular_pipelines/flux3/decoders.py index ce30073bc544..a8486993e823 100644 --- a/src/diffusers/modular_pipelines/flux3/decoders.py +++ b/src/diffusers/modular_pipelines/flux3/decoders.py @@ -47,15 +47,17 @@ def inputs(self) -> list[tuple[str, Any]]: type_hint=torch.Tensor, description="The denoised image latent tokens from the denoising step. Shape: (B, seq_len, C)", ), - # no default: the size is already baked into the latents, every workflow sets these upstream + # required, no default: the size is already baked into the latents, every workflow sets these upstream InputParam( "height", type_hint=int, + required=True, description="The height in pixels of the generated image; must match the size the latents were prepared at", ), InputParam( "width", type_hint=int, + required=True, description="The width in pixels of the generated image; must match the size the latents were prepared at", ), ] diff --git a/src/diffusers/modular_pipelines/ltx/denoise.py b/src/diffusers/modular_pipelines/ltx/denoise.py index d04c0c7f18d5..1107bb1a3463 100644 --- a/src/diffusers/modular_pipelines/ltx/denoise.py +++ b/src/diffusers/modular_pipelines/ltx/denoise.py @@ -99,9 +99,9 @@ def inputs(self) -> list[tuple[str, Any]]: InputParam.template("attention_kwargs"), InputParam.template("num_inference_steps", required=True, default=None), InputParam("rope_interpolation_scale", type_hint=tuple), - InputParam.template("height"), - InputParam.template("width"), - InputParam("num_frames", type_hint=int), + InputParam.template("height", required=True), + InputParam.template("width", required=True), + InputParam("num_frames", type_hint=int, required=True), ] guider_input_names = [] for value in self._guider_input_fields.values(): @@ -333,9 +333,9 @@ def inputs(self) -> list[tuple[str, Any]]: InputParam.template("attention_kwargs"), InputParam.template("num_inference_steps", required=True, default=None), InputParam("rope_interpolation_scale", type_hint=tuple), - InputParam.template("height"), - InputParam.template("width"), - InputParam("num_frames", type_hint=int), + InputParam.template("height", required=True), + InputParam.template("width", required=True), + InputParam("num_frames", type_hint=int, required=True), ] guider_input_names = [] for value in self._guider_input_fields.values(): @@ -413,9 +413,9 @@ def description(self) -> str: @property def inputs(self) -> list[InputParam]: return [ - InputParam.template("height"), - InputParam.template("width"), - InputParam("num_frames", type_hint=int), + InputParam.template("height", required=True), + InputParam.template("width", required=True), + InputParam("num_frames", type_hint=int, required=True), ] @torch.no_grad() diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py index 16b176ce5c88..b2b2ab0287c1 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py @@ -68,7 +68,7 @@ class LTX2AutoPromptEnhancerStep(ConditionalPipelineBlocks): prompt_enhancer (`PreTrainedModel`) processor (`ProcessorMixin`) Inputs: - prompt (`str`, *optional*): + prompt (`str`, *optional*, required, depending on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -87,7 +87,7 @@ class LTX2AutoPromptEnhancerStep(ConditionalPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. Outputs: @@ -198,11 +198,11 @@ class LTX2AutoDurationStep(ConditionalPipelineBlocks): Upper bound on the auto-predicted duration. Must be strictly greater than `min_seconds`. frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. - connector_prompt_embeds (`Tensor`, *optional*): + connector_prompt_embeds (`Tensor`, *optional*, required, depending on the workflow): Video-branch text conditioning from the connector (positive prompt). - connector_audio_prompt_embeds (`Tensor`, *optional*): + connector_audio_prompt_embeds (`Tensor`, *optional*, required, depending on the workflow): Audio-branch text conditioning from the connector (positive prompt). - batch_size (`int`, *optional*): + batch_size (`int`, *optional*, required, depending on the workflow): The number of prompts being denoised, used to expand conditioning per prompt. Outputs: @@ -238,7 +238,7 @@ class LTX2AutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLLTX2Video`) text_encoder (`PreTrainedModel`) video_processor (`VideoProcessor`) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -665,7 +665,7 @@ class LTX2AutoReferenceEncoderStep(ConditionalPipelineBlocks): vae (`AutoencoderKLLTX2Video`) transformer (`LTX2VideoTransformer3DModel`) video_processor (`VideoProcessor`) Inputs: - reference_conditions (`list`, *optional*): + reference_conditions (`list`, *optional*, required, depending on the workflow): `LTX2ReferenceCondition` (or list of them) whose videos are encoded into extra latent tokens the IC-LoRA adapter attends to. reference_downscale_factor (`int`, *optional*, defaults to 1): @@ -737,15 +737,15 @@ class LTX2AutoBuildVideoSelfAttentionMaskStep(ConditionalPipelineBlocks): - Skipped otherwise, leaving attention unmasked. Inputs: - latents (`Tensor`, *optional*): + latents (`Tensor`, *optional*, required, depending on the workflow): Pre-generated noisy latents for image generation. - base_token_count (`int`, *optional*): + base_token_count (`int`, *optional*, required, depending on the workflow): Number of generated-video tokens, i.e. the sequence length before appended tokens. - num_ref_tokens (`int`, *optional*): + num_ref_tokens (`int`, *optional*, required, depending on the workflow): Number of reference tokens, which sit at the very end of the sequence. - reference_cross_mask (`Tensor`, *optional*): + reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. - reference_token_counts (`list`, *optional*): + reference_token_counts (`list`, *optional*, required, depending on the workflow): Per-reference token counts, used to split `reference_cross_mask` into attention groups. Outputs: @@ -860,7 +860,7 @@ class LTX2InContextCoreDenoiseStep(SequentialPipelineBlocks): The number of prompts being denoised, used to expand conditioning per prompt. generator (`Generator`, *optional*): Torch generator for deterministic generation. - reference_cross_mask (`Tensor`, *optional*): + reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. @@ -953,13 +953,13 @@ class LTX2AutoCoreDenoiseStep(ConditionalPipelineBlocks): Audio-branch text conditioning (uncond). negative_connector_attention_mask (`Tensor`): Binary text attention mask (uncond). - condition_latents (`list`, *optional*): + condition_latents (`list`, *optional*, required, depending on the workflow): Per-condition normalized VAE latents of shape [1, C, F, H, W]. - condition_strengths (`list`, *optional*): + condition_strengths (`list`, *optional*, required, depending on the workflow): Per-condition conditioning strengths. - condition_indices (`list`, *optional*): + condition_indices (`list`, *optional*, required, depending on the workflow): Per-condition latent frame index at which the condition is applied. - condition_pixel_frames (`list`, *optional*): + condition_pixel_frames (`list`, *optional*, required, depending on the workflow): Per-condition trimmed pixel frame count, used to clamp single-frame keyframe coords. reference_conditions (`list`, *optional*): `LTX2ReferenceCondition` (or list of them); only their `strength` is read here. Omit for IC-LoRAs that @@ -995,7 +995,7 @@ class LTX2AutoCoreDenoiseStep(ConditionalPipelineBlocks): The number of prompts being denoised, used to expand conditioning per prompt. generator (`Generator`, *optional*): Torch generator for deterministic generation. - reference_cross_mask (`Tensor`, *optional*): + reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. @@ -1011,7 +1011,7 @@ class LTX2AutoCoreDenoiseStep(ConditionalPipelineBlocks): Whether to condition the transformer on a separate per-token cross timestep (LTX-2.3+). attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): VAE-encoded reference-image latents used for image-to-video conditioning. Outputs: @@ -1197,7 +1197,7 @@ class LTX2AutoDecoderStep(AutoPipelineBlocks): Inputs: latents (`Tensor`): Pre-generated noisy latents for image generation. - base_token_count (`int`, *optional*): + base_token_count (`int`, *optional*, required, depending on the workflow): Number of generated-video tokens, i.e. the sequence length before appended tokens. output_type (`str`, *optional*, defaults to pil): Output format: 'pil', 'np', 'pt'. @@ -1268,7 +1268,7 @@ class LTX2Blocks(SequentialPipelineBlocks): (`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*): + prompt (`str`, *optional*, required, depending on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1287,7 +1287,7 @@ class LTX2Blocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -1380,7 +1380,7 @@ class LTX2ImageToVideoBlocks(SequentialPipelineBlocks): audio_guider (`LTX2Guidance`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*): + prompt (`str`, *optional*, required, depending on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1399,7 +1399,7 @@ class LTX2ImageToVideoBlocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -1501,7 +1501,7 @@ class LTX2ConditionBlocks(SequentialPipelineBlocks): (`LTX2Guidance`) video_processor (`VideoProcessor`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*): + prompt (`str`, *optional*, required, depending on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1520,7 +1520,7 @@ class LTX2ConditionBlocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -1621,7 +1621,7 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): (`LTX2Guidance`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*): + prompt (`str`, *optional*, required, depending on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1640,7 +1640,7 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -1653,7 +1653,7 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): num_frames (`int`, *optional*): The number of frames in the generated video. Omit to auto-predict via the `duration_head` (see `LTX2AutoDurationStep`). - reference_conditions (`list`, *optional*): + reference_conditions (`list`, *optional*, required, depending on the workflow): `LTX2ReferenceCondition` (or list of them) whose videos are encoded into extra latent tokens the IC-LoRA adapter attends to. reference_downscale_factor (`int`, *optional*, defaults to 1): @@ -1689,7 +1689,7 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): `sigmas` are supplied, else 1.0. sigmas (`list`, *optional*): Custom sigmas for the denoising process. - reference_cross_mask (`Tensor`, *optional*): + reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. @@ -1777,7 +1777,7 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): audio_guider (`LTX2Guidance`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*): + prompt (`str`, *optional*, required, depending on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1796,7 +1796,7 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -1820,7 +1820,7 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): num_frames (`int`, *optional*): The number of frames in the generated video. Omit to auto-predict via the `duration_head` (see `LTX2AutoDurationStep`). - reference_conditions (`list`, *optional*): + reference_conditions (`list`, *optional*, required, depending on the workflow): `LTX2ReferenceCondition` (or list of them) whose videos are encoded into extra latent tokens the IC-LoRA adapter attends to. reference_downscale_factor (`int`, *optional*, defaults to 1): @@ -1840,13 +1840,13 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): slow motion. `frame_rate` stays the playback rate. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - condition_latents (`list`, *optional*): + condition_latents (`list`, *optional*, required, depending on the workflow): Per-condition normalized VAE latents of shape [1, C, F, H, W]. - condition_strengths (`list`, *optional*): + condition_strengths (`list`, *optional*, required, depending on the workflow): Per-condition conditioning strengths. - condition_indices (`list`, *optional*): + condition_indices (`list`, *optional*, required, depending on the workflow): Per-condition latent frame index at which the condition is applied. - condition_pixel_frames (`list`, *optional*): + condition_pixel_frames (`list`, *optional*, required, depending on the workflow): Per-condition trimmed pixel frame count, used to clamp single-frame keyframe coords. reference_latents (`Tensor`, *optional*): Packed reference tokens of shape [1, total_reference_tokens, C], or `None` when no reference conditions @@ -1862,7 +1862,7 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): `sigmas` are supplied, else 1.0. sigmas (`list`, *optional*): Custom sigmas for the denoising process. - reference_cross_mask (`Tensor`, *optional*): + reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. @@ -1876,8 +1876,10 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): Whether to condition the transformer on a separate per-token cross timestep (LTX-2.3+). attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): VAE-encoded reference-image latents used for image-to-video conditioning. + base_token_count (`int`, *optional*, required, depending on the workflow): + Number of generated-video tokens, i.e. the sequence length before appended tokens. output_type (`str`, *optional*, defaults to pil): Output format: 'pil', 'np', 'pt'. decode_timestep (`None`, *optional*, defaults to 0.0): diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py index 3b89ddc3bbb7..1c3a3d26f33b 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py @@ -15,7 +15,7 @@ import torch from ..modular_pipeline import AutoPipelineBlocks, SequentialPipelineBlocks -from ..modular_pipeline_utils import OutputParam +from ..modular_pipeline_utils import InputParam, OutputParam from .decoders import ( LTX2AudioDecoderStep, LTX2DiffusionVaeDecoderStep, @@ -162,7 +162,7 @@ class LTX25AutoDecoderStep(AutoPipelineBlocks): Inputs: latents (`Tensor`): Pre-generated noisy latents for image generation. - base_token_count (`int`, *optional*): + base_token_count (`int`, *optional*, required, depending on the workflow): Number of generated-video tokens, i.e. the sequence length before appended tokens. output_type (`str`, *optional*, defaults to pil): Output format: 'pil', 'np', 'pt'. @@ -233,7 +233,7 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): audio_guider (`LTX2Guidance`) diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*): + prompt (`str`, *optional*, required, depending on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -252,7 +252,7 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -276,7 +276,7 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): num_frames (`int`, *optional*): The number of frames in the generated video. Omit to auto-predict via the `duration_head` (see `LTX2AutoDurationStep`). - reference_conditions (`list`, *optional*): + reference_conditions (`list`, *optional*, required, depending on the workflow): `LTX2ReferenceCondition` (or list of them) whose videos are encoded into extra latent tokens the IC-LoRA adapter attends to. reference_downscale_factor (`int`, *optional*, defaults to 1): @@ -296,13 +296,13 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): slow motion. `frame_rate` stays the playback rate. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - condition_latents (`list`, *optional*): + condition_latents (`list`, *optional*, required, depending on the workflow): Per-condition normalized VAE latents of shape [1, C, F, H, W]. - condition_strengths (`list`, *optional*): + condition_strengths (`list`, *optional*, required, depending on the workflow): Per-condition conditioning strengths. - condition_indices (`list`, *optional*): + condition_indices (`list`, *optional*, required, depending on the workflow): Per-condition latent frame index at which the condition is applied. - condition_pixel_frames (`list`, *optional*): + condition_pixel_frames (`list`, *optional*, required, depending on the workflow): Per-condition trimmed pixel frame count, used to clamp single-frame keyframe coords. reference_latents (`Tensor`, *optional*): Packed reference tokens of shape [1, total_reference_tokens, C], or `None` when no reference conditions @@ -318,7 +318,7 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): `sigmas` are supplied, else 1.0. sigmas (`list`, *optional*): Custom sigmas for the denoising process. - reference_cross_mask (`Tensor`, *optional*): + reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. @@ -332,8 +332,10 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): Whether to condition the transformer on a separate per-token cross timestep (LTX-2.3+). attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): VAE-encoded reference-image latents used for image-to-video conditioning. + base_token_count (`int`, *optional*, required, depending on the workflow): + Number of generated-video tokens, i.e. the sequence length before appended tokens. output_type (`str`, *optional*, defaults to pil): Output format: 'pil', 'np', 'pt'. @@ -365,6 +367,19 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): "denoise", "decode", ] + # `condition_encoder` declares `num_frames` without a default (the duration head predicts it) and `decode` + # defaults it to 121, so the pipeline declares it itself: no default, auto-predicted when omitted + input_overrides = { + "num_frames": InputParam( + "num_frames", + type_hint=int, + default=None, + description=( + "The number of frames in the generated video. Omit to auto-predict via the `duration_head` " + "(see `LTX2AutoDurationStep`)." + ), + ) + } # `num_frames` on `in_context` is a requirement, not a trigger: the in-context checkpoints ship without a # `duration_head`, so the workflow drops `LTX2AutoDurationStep` and `LTX2ConditionEncoderStep` raises if # `num_frames` is still `None`. Matches `LTX2InContextBlocks`, which omits the duration step outright. diff --git a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py index 318f55dd1506..b8a2d3a572b1 100644 --- a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py +++ b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py @@ -15,7 +15,7 @@ import torch from ..modular_pipeline import ConditionalPipelineBlocks, SequentialPipelineBlocks -from ..modular_pipeline_utils import OutputParam +from ..modular_pipeline_utils import InputParam, OutputParam from .before_denoise import ( MiniMaxH3FL2VAPrepareLatentsStep, MiniMaxH3NoKeyframeAnchorsStep, @@ -56,7 +56,7 @@ class MiniMaxH3AutoBeforeEncodeStep(ConditionalPipelineBlocks): 2048) Inputs: - references (`list`, *optional*): + references (`list`, *optional*, required, depending on the workflow): The references to condition on, **in the order the model should read them**: the order labels them in the prompt presentation and lays them out on the shared rotary clock, so a different order is a different request. One dataclass per modality, all holding in-memory media — a [`MiniMaxH3ImageReference`] (at most @@ -68,7 +68,7 @@ class MiniMaxH3AutoBeforeEncodeStep(ConditionalPipelineBlocks): Height of the generated video in pixels, a multiple of 32. width (`int`, *optional*): Width of the generated video in pixels, a multiple of 32. - num_frames (`int`, *optional*): + num_frames (`int`, *optional*, required, depending on the workflow): Number of frames to generate, at the fixed 24 fps. Snapped up to the next `17 * n + 5` the video VAE can decode; the resulting duration must stay between 5 and 15 seconds. To generate a video as long as a reference soundtrack, pass `round(samples / sample_rate * 24)`. @@ -140,7 +140,7 @@ class MiniMaxH3AutoTextEncoderStep(ConditionalPipelineBlocks): Inputs: prompt (`str`): The prompt to guide generation, a single string. - normalized_references (`list`, *optional*): + normalized_references (`list`, *optional*, required, depending on the workflow): The references normalized by the setup step, in packed order. keyframes (`list`, *optional*): The keyframes put onto the target canvas, in packed order. @@ -191,9 +191,9 @@ class MiniMaxH3AutoVaeEncoderStep(ConditionalPipelineBlocks): vae (`AutoencoderKLMiniMaxH3`) audio_vae (`AutoencoderKLMiniMaxH3Audio`) Inputs: - normalized_references (`list`, *optional*): + normalized_references (`list`, *optional*, required, depending on the workflow): The references normalized by the setup step, in packed order. - keyframes (`list`, *optional*): + keyframes (`list`, *optional*, required, depending on the workflow): The keyframes put onto the target canvas, in packed order. Outputs: @@ -543,18 +543,18 @@ class MiniMaxH3AutoDenoiseStep(ConditionalPipelineBlocks): Inputs: text_token_tags (`Tensor`): The per-row modality tag of every row of `prompt_embeds`. - normalized_references (`list`, *optional*): + normalized_references (`list`, *optional*, required, depending on the workflow): The references normalized by the setup step, in packed order. - condition_latents (`list`, *optional*): + condition_latents (`list`, *optional*, required, depending on the workflow): The encoded video conditioning latents, one per image and video reference in packed order. Their shape is where every reference block's geometry comes from. - audio_condition_latents (`list`, *optional*): + audio_condition_latents (`list`, *optional*, required, depending on the workflow): The encoded audio conditioning rows, one per audio-bearing reference in packed order. - height (`int`, *optional*): + height (`int`, *optional*, required or defaults to None, depending on the workflow): Height of the generated video in pixels. - width (`int`, *optional*): + width (`int`, *optional*, required or defaults to None, depending on the workflow): Width of the generated video in pixels. - num_frames (`int`, *optional*, defaults to None or 124, depending on the workflow): + num_frames (`int`, *optional*, required or defaults to 124, depending on the workflow): Resolved number of frames, of the form 17 * n + 5. generator (`Generator`, *optional*): The generator of the request. The conditioning noise is drawn from it first, one draw per condition, @@ -680,7 +680,7 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): 2048) Inputs: - references (`list`, *optional*): + references (`list`, *optional*, required, depending on the workflow): The references to condition on, **in the order the model should read them**: the order labels them in the prompt presentation and lays them out on the shared rotary clock, so a different order is a different request. One dataclass per modality, all holding in-memory media — a [`MiniMaxH3ImageReference`] (at most @@ -692,7 +692,7 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): Height of the generated video in pixels, a multiple of 32. width (`int`, *optional*): Width of the generated video in pixels, a multiple of 32. - num_frames (`int`, *optional*, defaults to None or 124, depending on the workflow): + num_frames (`int`, *optional*): Number of frames to generate, at the fixed 24 fps. Snapped up to the next `17 * n + 5` the video VAE can decode; the resulting duration must stay between 5 and 15 seconds. To generate a video as long as a reference soundtrack, pass `round(samples / sample_rate * 24)`. @@ -704,14 +704,14 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): it is the follower of the two and is cover-cropped onto the canvas. prompt (`str`): The prompt to guide generation, a single string. - normalized_references (`list`, *optional*): + normalized_references (`list`, *optional*, required, depending on the workflow): The references normalized by the setup step, in packed order. keyframes (`list`, *optional*): The keyframes put onto the target canvas, in packed order. - condition_latents (`list`, *optional*): + condition_latents (`list`, *optional*, required, depending on the workflow): The encoded video conditioning latents, one per image and video reference in packed order. Their shape is where every reference block's geometry comes from. - audio_condition_latents (`list`, *optional*): + audio_condition_latents (`list`, *optional*, required, depending on the workflow): The encoded audio conditioning rows, one per audio-bearing reference in packed order. generator (`Generator`, *optional*): The generator of the request. The conditioning noise is drawn from it first, one draw per condition, @@ -757,6 +757,19 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): "denoise", "decode", ] + # `ref2va` requires `num_frames` (the references are truncated to it) while `t2va` and `fl2va` default it to 124, + # so the pipeline declares it itself: optional, no default, each workflow keeps its own + input_overrides = { + "num_frames": InputParam( + name="num_frames", + type_hint=int, + description=( + "Number of frames to generate, at the fixed 24 fps. Snapped up to the next `17 * n + 5` the video " + "VAE can decode; the resulting duration must stay between 5 and 15 seconds. To generate a video " + "as long as a reference soundtrack, pass `round(samples / sample_rate * 24)`." + ), + ) + } # One repository holds both checkpoint partitions (`transformer/` for `t2va`/`fl2va`, `transformer_ref/` for # `ref2va`), so the tasks are workflows over one shared `modular_model_index.json`, each declaring only the # components its own half loads. The keys below are the task names a `workflow=` argument to diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index f6ee73dff153..36c4333ba89d 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -19,7 +19,7 @@ import traceback import warnings from collections import OrderedDict -from copy import deepcopy +from copy import copy, deepcopy from dataclasses import dataclass, field, replace from typing import Any @@ -53,13 +53,13 @@ from .components_manager import ComponentsManager from .modular_pipeline_utils import ( MODULAR_MODEL_CARD_TEMPLATE, + REQUIRED, ComponentSpec, ConfigSpec, InputParam, InsertableDict, OutputParam, _validate_requirements, - combine_inputs, combine_outputs, format_components, format_configs, @@ -770,34 +770,82 @@ def expected_configs(self): expected_configs.append(config) return override_attribute(expected_configs, self.config_overrides, "config_overrides", self.__class__.__name__) - @property - def required_inputs(self) -> list[str]: - # no default block means this conditional block can be skipped entirely - if self.default_block_name is None: - return [] - - first_block = next(iter(self.sub_blocks.values())) - required_by_all = set(getattr(first_block, "required_inputs", set())) + def _get_inputs(self): + # Merge the sub-blocks' inputs into one. Each sub-block is a branch and only one of them runs, so an input declared by + # several sub-blocks under the same name (for kwargs inputs, the same `kwargs_type`) is combined into one input: + # - its `type_hint` and `description` are defined by the first declaration. + # - `required` and `default` are only meaningful when the branches agree on them. they agree when: + # - (1) every sub-block requires the input, and the block cannot be skipped: the merged input is required + # - (2) every sub-block that declares it does so as optional, with the exact same default: the merged input is + # optional with that default + # - otherwise (some require it and others do not, the defaults differ, or all of them require it but the block + # itself can be skipped): the merged input is optional with default `None`, and `defaults_by_block` records + # what each declaring sub-block says. At runtime the sub-block that runs fills in its own default, or raises when + # it requires the input and it is missing (see `ModularPipelineBlocks.get_block_state`); + # the docstrings render the defaults, e.g. "defaults to 0.3 or 0.9999, depending on the workflow" + + declarations = {} + # declarations: input_name -> {block_name: InputParam}, `None` when the sub-block does not declare the input + # e.g. {"strength": {"inpaint": InputParam("strength", default=0.9999), "img2img": InputParam(...), + # "text2img": None}} + for block_name, block in self.sub_blocks.items(): + for input_param in block.inputs: + # kwargs inputs have no name, key them by "*_" + if input_param.name is None and input_param.kwargs_type is not None: + input_name = "*_" + input_param.kwargs_type + else: + input_name = input_param.name + # every sub-block gets a slot, in order, so a silent sub-block shows up as None + by_block = declarations.setdefault(input_name, dict.fromkeys(self.sub_blocks)) + if by_block[block_name] is None: + by_block[block_name] = input_param + + # assemble the final merged input param list + inputs = [] + for input_name, by_block in declarations.items(): + # keep the first `InputParam` for type hint and description + input_param = next(p for p in by_block.values() if p is not None) + + # required only when every sub-block requires it and the user cannot skip the whole block + required = ( + all(p is not None and p.required for p in by_block.values()) and self.default_block_name is not None + ) + # same default when every declaring sub-block says optional with the same default. A silent sub-block does not + # declare the input, so it agrees with whatever its siblings say. A required sub-block and a nested conditional + # block carrying a record of its own both have `default=None`, so they are ruled out explicitly, otherwise + # they would match a sibling that is optional with no default + same_default = all( + p is None or (not p.required and not p.defaults_by_block and p.default == input_param.default) + for p in by_block.values() + ) + if required or same_default: + inputs.append(input_param) + continue - # Intersect with required inputs from all other blocks - for block in list(self.sub_blocks.values())[1:]: - block_required = set(getattr(block, "required_inputs", set())) - required_by_all.intersection_update(block_required) + # the branches disagree: record what each declaring sub-block says, its default or `REQUIRED` when it + # requires the input, e.g. {"img2img": 0.3, "inpaint": 0.9999} for `strength`, or + # {"t2va": 124, "ref2va": REQUIRED} for `num_frames`. A silent sub-block has no entry + defaults_by_block = {} + for block_name, inp in by_block.items(): + if inp is None: + continue + if inp.defaults_by_block: + # nested conditional block: prefix its block names with the sub-block name, e.g. from sub-block + # "image", {"img2img": 0.3, "inpaint": 0.9999} -> {"image.img2img": 0.3, "image.inpaint": 0.9999} + defaults_by_block.update({f"{block_name}.{k}": v for k, v in inp.defaults_by_block.items()}) + else: + # one entry for a leaf sub-block: e.g. {"img2img": 0.3} + defaults_by_block[block_name] = REQUIRED if inp.required else inp.default + # a merged input with a record always has `default=None` and `required=False`: the sub-block that runs fills + # in its own default at runtime, or raises when it requires the input and it is missing. `replace` creates + # a fresh copy so the sub-blocks' own params are never mutated + inputs.append(replace(input_param, default=None, required=False, defaults_by_block=defaults_by_block)) - return list(required_by_all) + return override_attribute(inputs, self.input_overrides, "input_overrides", self.__class__.__name__) @property def inputs(self) -> list[tuple[str, Any]]: - named_inputs = [(name, block.inputs) for name, block in self.sub_blocks.items()] - combined_inputs = combine_inputs(*named_inputs) - # required only if every sub-block requires it; - for position, input_param in enumerate(combined_inputs): - if input_param.name in self.required_inputs: - # `replace` copies so the sub-blocks' declarations stay untouched - combined_inputs[position] = replace(input_param, required=True) - else: - combined_inputs[position] = replace(input_param, required=False) - return override_attribute(combined_inputs, self.input_overrides, "input_overrides", self.__class__.__name__) + return self._get_inputs() @property def intermediate_outputs(self) -> list[str]: @@ -1230,96 +1278,74 @@ def __init__(self): def _get_inputs(self): # Walk the sub-blocks in order to derive the pipeline inputs. An input is listed unless an earlier block # produces it. The first declaration is kept; a later declaration of the same input can only make it required - # or disagree on its default. + # or warn about a conflict in the defaults. # - # The comments below follow one img2img pipeline: + # The comments below follow one img2img pipeline as example: # `text_encoder -> image_encoder -> set_timesteps -> prepare_latents -> denoise`, where `image_encoder` is a - # ConditionalPipelineBlocks without a default block: it runs when `image` is passed and produces - # `image_latents`. - inputs = {} # pipeline inputs + # ConditionalPipelineBlocks which runs only when `image` is passed and produces `image_latents`. + inputs = {} # input_name -> InputParam, e.g. {"num_inference_steps": InputParam("num_inference_steps", default=50)} outputs = set() # produced by an earlier block - maybe_outputs = set() # produced by an earlier ConditionalPipelineBlocks, if it runs + # maybe produced by an earlier block: a ConditionalPipelineBlocks that can be skipped, or whose branches do not + # all produce the output + maybe_outputs = set() for block_name, block in self.sub_blocks.items(): for inp in block.inputs: - # kwargs inputs have no name, key them by "*_" like `combine_inputs` does + # kwargs inputs have no name, key them by "*_" key = inp.name if inp.name is not None else f"*_{inp.kwargs_type}" if key in outputs: - # produced by an earlier block, so not a pipeline input: - # e.g. `denoise` needs `prompt_embeds`, but `text_encoder` produces it + # not a pipeline input if it'sproduced by an earlier block + # e.g. `prompt` -> `text_encoder` -> `prompt_embeds` -> `denoise` + # here `prompt_embeds` is an input for `denoise` but not an input for pipeline because `text_encoder` outputs it continue - # first declaration, listed as declared: - # e.g. `set_timesteps` declares `num_inference_steps` and no earlier block declared or produced it, so - # it becomes a pipeline input with the default `set_timesteps` gave it if key not in inputs: - # `replace` copies: the flags below are the composite's view and must not reach the sub-block - inputs[key] = replace(inp) + # otherwise, declare it as an input if not already declared + inputs[key] = copy(inp) if inp.required and key in maybe_outputs: - # the block requires it, but an earlier conditional block can produce it, so the pipeline lists - # it optional: - # e.g. `prepare_latents` requires `image_latents`, but `image_encoder` produces it when `image` - # is passed, so `image_latents` is optional for the pipeline: the user can pass `image` instead + # the block requires it, but an earlier conditional block can produce it, so change it to optional + # e.g. `prepare_latents` lists `image_latents` as a required input, but `image_encoder` (an conditional block) produces it when `image` + # is passed, so `image_latents` is an optional input for the pipeline, i.e. user can skip it if they pass `image` and run the image_encoder block. inputs[key].required = False continue - if inp.required: - if inputs[key].default is None and not inputs[key].defaults_by_block and key not in maybe_outputs: - # already listed as optional with no default, nothing earlier can produce it, and now a block - # requires it, so the user has to pass it: - # e.g. `prepare_latents` declares `height` optional, `denoise` requires it, so `height` is a - # required pipeline input. Had `prepare_latents` given `height` a default, or had the branches - # of an earlier conditional block each given it theirs, it would stay optional - inputs[key].required = True - elif not inputs[key].required: - if inputs[key].default is None and not inputs[key].defaults_by_block and inp.defaults_by_block: - # the entry has no default and this conditional block's branches each have their own: carry - # them up, prefixed like `combine_inputs` does, so the pipeline docstring shows them. Not a - # conflict: the branch that runs fills in its own default when it reads the state - # (`get_block_state`): - # e.g. `before_encode` lists `num_frames` with no default and the `denoise` branches default - # it to 124 or require it, so the pipeline says "None or 124, depending on the workflow" - inputs[key].defaults_by_block = { - f"{block_name}.{k}": v for k, v in inp.defaults_by_block.items() - } - elif key not in self.input_overrides: - # the defaults each side has an opinion on: its own, or the per-branch ones a conditional block - # recorded. `None` means no default, not an opinion, so it is left out on both sides - existing, declared = inputs[key], inp - existing_defaults = ( - [existing.default] - if existing.default is not None - else [d for d in (existing.defaults_by_block or {}).values() if d is not None] - ) - declared_defaults = ( - [declared.default] - if declared.default is not None - else [d for d in (declared.defaults_by_block or {}).values() if d is not None] + # if the input is already declared: a later declaration can only make it required, or warn on a conflict in the defaults + existing = inputs[key] + if inp.required and not existing.required: + # a later block requires an input currently listed as optional. + # we only change it to required if: + # (1) it is not possible for an earlier conditional block to produce it + # (2) no earlier default fills it in: the existing default is `None`, and if it comes from a + # conditional block with per-branch defaults, no branch declares a non-None one + # e.g. `prepare_latents` declares `height` optional with `None` default, `denoise` requires it, so + # `height` is a required pipeline input. Had `prepare_latents` defaults `height` to 512, it would + # stay optional with the default + has_default = existing.default is not None or any( + d is not None and d is not REQUIRED for d in (existing.defaults_by_block or {}).values() + ) + if key not in maybe_outputs and not has_default: + existing.required = True + elif not inp.required and not existing.required and key not in self.input_overrides: + # if two blocks both declare the same input as optional, with different non-None defaults, + # use the first one, but warn and ask user to set the `input_overrides` on the composite to explicitly set the default. + # e.g. `strength` is 0.3 on `set_timesteps` and 0.9999 on `prepare_latents`: the pipeline uses + # 0.3, and `input_overrides` on the composite picks another value + if inp.default is not None and inp.default != existing.default: + existing_text = f"default {existing.default!r}" + if existing.defaults_by_block: + existing_text = f"per-branch defaults {existing.defaults_by_block!r}" + logger.warning( + f"{self.__class__.__name__}: sub-block '{block_name}' declares default {inp.default!r} for " + f"input '{key}', but an earlier sub-block declares {existing_text}. The sequence keeps the " + f"earlier declaration. Declare '{key}' in `input_overrides` on {self.__class__.__name__} " + f"if that is not intended." ) - # (`in` compares with `==`, so list defaults such as `[2, 2, 2]` work too) - if (existing_defaults and any(d not in existing_defaults for d in declared_defaults)) or ( - declared_defaults and any(d not in declared_defaults for d in existing_defaults) - ): - # two optional declarations disagree on the default. In a sequence the input is one shared - # value, so the composite has to pick it with `input_overrides`; until 1.0.0 the first - # declaration wins: - # e.g. `strength` is 0.3 on `set_timesteps` and 0.9999 on `prepare_latents`, so the - # pipeline uses 0.3 and warns until it declares `strength` in `input_overrides` - existing_text = f"default {existing.default!r}" - if existing.defaults_by_block: - existing_text = f"per-branch defaults {existing.defaults_by_block!r}" - declared_text = f"default {declared.default!r}" - if declared.defaults_by_block: - declared_text = f"per-branch defaults {declared.defaults_by_block!r}" - deprecate( - "sub-block default conflict", - "1.0.0", - f"{self.__class__.__name__}: sub-block '{block_name}' declares {declared_text} for " - f"input '{key}' but an earlier sub-block declares {existing_text}. The first " - f"declaration currently wins; this will become an error. Declare the input in " - f"`input_overrides` on {self.__class__.__name__} to pick the default explicitly.", - standard_warn=False, - ) - if isinstance(block, ConditionalPipelineBlocks) and block.default_block_name is None: - # e.g. `image_encoder` may be skipped, so `image_latents` is only produced if it runs: a "maybe" output - maybe_outputs.update(out.name for out in block.intermediate_outputs) + if isinstance(block, ConditionalPipelineBlocks): + # for conditional blocks, + # if the output is produced by every branch and the block itself cannot be skipped, add it to the `outputs` + # otherwise, add it to the `maybe_outputs` + # e.g. `image_encoder` may be skipped, so `image_latents` is only produced if it runs, a maybe_outputs + per_branch = [{out.name for out in sub.intermediate_outputs} for sub in block.sub_blocks.values()] + produced = set.intersection(*per_branch) if block.default_block_name is not None else set() + outputs.update(produced) + maybe_outputs.update(set().union(*per_branch) - produced) else: outputs.update(out.name for out in block.intermediate_outputs) @@ -1626,42 +1652,36 @@ def _get_inputs(self): inputs = {} outputs = set() # produced by an earlier block for inp in self.loop_inputs: - # kwargs inputs have no name, key them by "*_" like `combine_inputs` does + # kwargs inputs have no name, key them by "*_" like `ConditionalPipelineBlocks` does key = inp.name if inp.name is not None else f"*_{inp.kwargs_type}" - inputs[key] = replace(inp) + inputs[key] = copy(inp) for block_name, block in self.sub_blocks.items(): for inp in block.inputs: key = inp.name if inp.name is not None else f"*_{inp.kwargs_type}" if key in outputs: continue if key not in inputs: - # `replace` copies: the flags below are the composite's view and must not reach the sub-block - inputs[key] = replace(inp) + # `copy`: the flags below are the composite's view and must not reach the sub-block + inputs[key] = copy(inp) continue if inp.required: if inputs[key].default is None: # nothing earlier defaults or produces it: the caller has to pass it inputs[key].required = True elif ( - inp.default is not None - and inputs[key].default is not None - and not inputs[key].required + not inputs[key].required + and inp.default is not None and inp.default != inputs[key].default and key not in self.input_overrides ): - # two optional declarations disagree on the default. In a sequence the input is one shared value, - # so the composite has to pick it with `input_overrides`; until 1.0.0 the first declaration wins. - # `None` means no default, not an opinion. Loop sub-blocks are leaves, so there are no per-branch - # defaults to consider here - deprecate( - "sub-block default conflict", - "1.0.0", + # two optional declarations: the earlier one wins, a later block that declares an actual default + # which is not the one in use gets a warning (a later `None` takes whatever is set). Loop + # sub-blocks are leaves, so there are no per-branch defaults to consider + logger.warning( f"{self.__class__.__name__}: sub-block '{block_name}' declares default {inp.default!r} for " - f"input '{key}' but an earlier declaration (a loop input or an earlier sub-block) has " - f"{inputs[key].default!r}. The first declaration currently wins; this will become an error. " - f"Declare the input in `input_overrides` on {self.__class__.__name__} to pick the default " - f"explicitly.", - standard_warn=False, + f"input '{key}', but an earlier declaration (a loop input or an earlier sub-block) has " + f"{inputs[key].default!r}. The loop keeps the earlier declaration. Declare '{key}' in " + f"`input_overrides` on {self.__class__.__name__} if that is not intended." ) outputs.update(out.name for out in block.intermediate_outputs) diff --git a/src/diffusers/modular_pipelines/modular_pipeline_utils.py b/src/diffusers/modular_pipelines/modular_pipeline_utils.py index 414e461bfbe4..5e495c51404c 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline_utils.py +++ b/src/diffusers/modular_pipelines/modular_pipeline_utils.py @@ -15,7 +15,7 @@ import inspect import re from collections import OrderedDict -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from types import UnionType from typing import Any, Literal, Type, Union, get_args, get_origin @@ -555,6 +555,20 @@ class ConfigSpec: } +class _Required: + """The marker `ConditionalPipelineBlocks` records in `defaults_by_block` for a sub-block requiring the input.""" + + def __repr__(self): + return "required" + + def __reduce__(self): + # keep the single instance through `copy`, `deepcopy` and pickling, so `is REQUIRED` stays true + return "REQUIRED" + + +REQUIRED = _Required() + + @dataclass class InputParam: """Specification for an input parameter.""" @@ -566,9 +580,12 @@ class InputParam: description: str = "" kwargs_type: str = None metadata: dict[str, Any] = None - # set by `combine_inputs` when sub-blocks of a ConditionalPipelineBlocks declare different defaults for this - # input: maps block name -> that block's declared default; `default` is None in that case and each block - # resolves its own default at runtime in `get_block_state` + # Set by `ConditionalPipelineBlocks._get_inputs` when its sub-blocks do not declare the same default for this + # input (different values, or one declares it required and another does not): maps block name -> that block's + # default, with `REQUIRED` standing for "no default, must be passed". `default` is None and `required` is False in + # that case, and each block resolves its own at runtime in `get_block_state`: it fills in its default, or raises + # when an input it requires is missing. Nothing at runtime reads the record; the docstring renderer and the + # `_get_inputs` of an enclosing composite block do defaults_by_block: dict[str, Any] = None def __post_init__(self): @@ -731,12 +748,15 @@ def wrap_text(text, indent, max_length): if hasattr(param, "required"): if not param.required: param_str += ", *optional*" - if param.defaults_by_block is not None: - # e.g. ", defaults to None or 189, depending on the workflow" - distinct_defaults = list(dict.fromkeys(param.defaults_by_block.values())) - param_str += ( - f", defaults to {' or '.join(str(v) for v in distinct_defaults)}, depending on the workflow" - ) + if param.defaults_by_block: + # "required" first when a branch requires it, then one entry per distinct default: + # e.g. {"a": None, "b": 189, "c": 189} -> ", defaults to None or 189, depending on the workflow" + # {"t2v": 124, "ref2v": REQUIRED} -> ", required or defaults to 124, depending on the workflow" + distinct = list(dict.fromkeys(param.defaults_by_block.values())) + parts = ["required"] if REQUIRED in distinct else [] + if any(v is not REQUIRED for v in distinct): + parts.append(f"defaults to {' or '.join(str(v) for v in distinct if v is not REQUIRED)}") + param_str += f", {' or '.join(parts)}, depending on the workflow" elif param.default is not None: param_str += f", defaults to {param.default}" param_str += "):" @@ -811,12 +831,13 @@ def get_type_str(type_hint): if hasattr(param, "required") and not param.required: param_str += ", *optional*" - if param.defaults_by_block is not None: - # e.g. ", defaults to `None` or `189`, depending on the workflow" - distinct_defaults = list(dict.fromkeys(param.defaults_by_block.values())) - param_str += ( - f", defaults to {' or '.join(f'`{v}`' for v in distinct_defaults)}, depending on the workflow" - ) + if param.defaults_by_block: + # e.g. {"t2v": 124, "ref2v": REQUIRED} -> ", required or defaults to `124`, depending on the workflow" + distinct = list(dict.fromkeys(param.defaults_by_block.values())) + parts = ["required"] if REQUIRED in distinct else [] + if any(v is not REQUIRED for v in distinct): + parts.append(f"defaults to {' or '.join(f'`{v}`' for v in distinct if v is not REQUIRED)}") + param_str += f", {' or '.join(parts)}, depending on the workflow" elif param.default is not None: param_str += f", defaults to `{param.default}`" param_str += ")" @@ -1089,70 +1110,6 @@ def _accumulate(mapping: dict[str, Any]): return normalized -def combine_inputs(*named_input_lists: list[tuple[str, list[InputParam]]]) -> list[InputParam]: - """ - Combines multiple lists of InputParam objects from different blocks. Duplicate inputs keep the first occurrence. If - duplicate inputs declare different defaults, the combined input's default is None and the per-block defaults are - recorded in `defaults_by_block`; the branch that runs fills in its own default when it reads the state - (`get_block_state`), and the docstring formatters render the per-block defaults. - - Example: - - ```python - combine_inputs( - ("img2img", [InputParam("prompt"), InputParam("strength", default=0.3)]), - ("inpaint", [InputParam("prompt"), InputParam("strength", default=0.9999)]), - ) - # returns: - # InputParam("prompt") # no disagreement -> first occurrence kept as-is - # InputParam("strength", default=None, defaults_by_block={"img2img": 0.3, "inpaint": 0.9999}) - ``` - - See `TestConditionalBlocksInputs` in `tests/modular_pipelines/test_conditional_pipeline_blocks.py` for the full - behavior. - - Args: - named_input_lists: List of tuples containing (block_name, input_param_list) pairs - - Returns: - List[InputParam]: Combined list of unique InputParam objects - """ - combined_dict = {} # name -> InputParam, e.g. {"strength": InputParam("strength", default=0.3)} - defaults_by_block = {} # name -> {block_name: default}, e.g. {"strength": {"img2img": 0.3, "inpaint": 0.9999}} - - for block_name, inputs in named_input_lists: - for input_param in inputs: - if input_param.name is None and input_param.kwargs_type is not None: - input_name = "*_" + input_param.kwargs_type - else: - input_name = input_param.name - - if input_param.defaults_by_block: - # nested conditional block: prefix its block names with the sub-block name, - # e.g. {"img2img": 0.3, "inpaint": 0.9999} from sub-block "image" -> {"image.img2img": 0.3, "image.inpaint": 0.9999} - new_defaults = {f"{block_name}.{k}": v for k, v in input_param.defaults_by_block.items()} - else: - new_defaults = {block_name: input_param.default} - - if input_name not in combined_dict: - combined_dict[input_name] = input_param - defaults_by_block[input_name] = new_defaults - continue - - # duplicate input: accumulate this block's default, - # e.g. "strength" was {"img2img": 0.3}, after inpaint's turn it is {"img2img": 0.3, "inpaint": 0.9999} - defaults_by_block[input_name].update(new_defaults) - defaults = list(defaults_by_block[input_name].values()) - if any(d != defaults[0] for d in defaults[1:]): - # blocks disagree (0.3 vs 0.9999): combined default becomes None and the per-block defaults are - # recorded on the param; `replace` creates a fresh copy so the blocks' own params are never mutated - combined_dict[input_name] = replace( - combined_dict[input_name], default=None, defaults_by_block=dict(defaults_by_block[input_name]) - ) - - return list(combined_dict.values()) - - def combine_outputs(*named_output_lists: list[tuple[str, list[OutputParam]]]) -> list[OutputParam]: """ Combines multiple lists of OutputParam objects from different blocks. For duplicate outputs, keeps the first @@ -1202,7 +1159,7 @@ class as the entry it replaces, takes that entry's place whole """ if not overrides: return items - # kwargs inputs/outputs have no name and are keyed by "*_", like `combine_inputs` does + # kwargs inputs/outputs have no name and are keyed by "*_", like `ConditionalPipelineBlocks` does index = { item.name if item.name is not None else f"*_{item.kwargs_type}": position for position, item in enumerate(items) diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py index 45fbeebdb239..f0ef07948570 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py @@ -332,15 +332,15 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): TODO: Add description. clip_skip (`None`, *optional*): TODO: Add description. - ip_adapter_image (`Image | ndarray | Tensor | list | list | list`, *optional*): + ip_adapter_image (`Image | ndarray | Tensor | list | list | list`, *optional*, required, depending on the workflow): The image(s) to be used as ip adapter height (`None`, *optional*): TODO: Add description. width (`None`, *optional*): TODO: Add description. - image (`None`, *optional*): + image (`None`, *optional*, required, depending on the workflow): TODO: Add description. - mask_image (`None`, *optional*): + mask_image (`None`, *optional*, required, depending on the workflow): TODO: Add description. padding_mask_crop (`None`, *optional*): TODO: Add description. @@ -371,10 +371,10 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): TODO: Add description. latents (`None`, *optional*): TODO: Add description. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): The latents representing the reference image for image-to-image/inpainting generation. Can be generated in vae_encode step. - mask (`Tensor`, *optional*): + mask (`Tensor`, *optional*, required, depending on the workflow): The mask for the inpainting generation. Can be generated in vae_encode step. masked_image_latents (`Tensor`, *optional*): The masked image latents for the inpainting generation (only for inpainting-specific unet). Can be @@ -395,9 +395,9 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): TODO: Add description. negative_aesthetic_score (`None`, *optional*, defaults to 2.0): TODO: Add description. - control_image (`None`, *optional*): + control_image (`None`, *optional*, required, depending on the workflow): TODO: Add description. - control_mode (`None`, *optional*): + control_mode (`None`, *optional*, required, depending on the workflow): TODO: Add description. control_guidance_start (`None`, *optional*, defaults to 0.0): TODO: Add description. @@ -410,12 +410,12 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): crops_coords (`tuple | NoneType`, *optional*): The crop coordinates to use for preprocess/postprocess the image and mask, for inpainting task only. Can be generated in vae_encode step. - controlnet_cond (`Tensor`, *optional*): + controlnet_cond (`Tensor`, *optional*, required, depending on the workflow): The control image to use for the denoising process. Can be generated in prepare_controlnet_inputs step. conditioning_scale (`float`, *optional*): The controlnet conditioning scale value to use for the denoising process. Can be generated in prepare_controlnet_inputs step. - controlnet_keep (`list`, *optional*): + controlnet_keep (`list`, *optional*, required, depending on the workflow): The controlnet keep values to use for the denoising process. Can be generated in prepare_controlnet_inputs step. **denoiser_input_fields (`None`, *optional*): @@ -430,6 +430,8 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): created and added to the pipeline state eta (`None`, *optional*, defaults to 0.0): TODO: Add description. + noise (`Tensor | NoneType`, *optional*): + The noise added to the image latents, for inpainting task only. Can be generated in prepare_latent step. output_type (`None`, *optional*, defaults to pil): TODO: Add description. diff --git a/src/diffusers/modular_pipelines/wan/encoders.py b/src/diffusers/modular_pipelines/wan/encoders.py index 6eaf42ce4f6a..9102d2f0e853 100644 --- a/src/diffusers/modular_pipelines/wan/encoders.py +++ b/src/diffusers/modular_pipelines/wan/encoders.py @@ -319,6 +319,9 @@ def inputs(self) -> list[InputParam]: def intermediate_outputs(self) -> list[OutputParam]: return [ OutputParam("resized_image", type_hint=PIL.Image.Image), + # the target area is kept, the aspect ratio comes from the image: the actual size is set here + OutputParam("height", type_hint=int, description="The height in pixels of the resized image."), + OutputParam("width", type_hint=int, description="The width in pixels of the resized image."), ] def __call__( diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py index 2570e341f1be..6f92a3085131 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py @@ -14,7 +14,7 @@ from ...utils import logging from ..modular_pipeline import SequentialPipelineBlocks -from ..modular_pipeline_utils import OutputParam +from ..modular_pipeline_utils import InputParam, OutputParam from .before_denoise import ( WanAdditionalInputsStep, WanPrepareLatentsStep, @@ -65,6 +65,10 @@ class WanImage2VideoVaeEncoderStep(SequentialPipelineBlocks): Outputs: resized_image (`Image`): TODO: Add description. + height (`int`): + The height in pixels of the resized image. + width (`int`): + The width in pixels of the resized image. first_frame_latents (`Tensor`): video latent representation with the first frame image condition image_condition_latents (`Tensor | NoneType`): @@ -192,7 +196,7 @@ class Wan22Image2VideoBlocks(SequentialPipelineBlocks): width (`int`, *optional*, defaults to 832): TODO: Add description. num_frames (`int`, *optional*, defaults to 81): - TODO: Add description. + The number of frames in the generated video. generator (`None`, *optional*): TODO: Add description. num_videos_per_prompt (`None`, *optional*, defaults to 1): @@ -230,6 +234,13 @@ class Wan22Image2VideoBlocks(SequentialPipelineBlocks): "denoise", "decode", ] + # `vae_encoder` defaults `num_frames` to 81 and the denoise step declares it without a default (it fills it in + # from the latents or the components), so the pipeline declares it itself + input_overrides = { + "num_frames": InputParam( + "num_frames", type_hint=int, default=81, description="The number of frames in the generated video." + ) + } @property def description(self): diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py index 01bbaef38ea7..06f6abd90e68 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py @@ -14,7 +14,7 @@ from ...utils import logging from ..modular_pipeline import AutoPipelineBlocks, SequentialPipelineBlocks -from ..modular_pipeline_utils import OutputParam +from ..modular_pipeline_utils import InputParam, OutputParam from .before_denoise import ( WanAdditionalInputsStep, WanPrepareLatentsStep, @@ -65,6 +65,10 @@ class WanImage2VideoImageEncoderStep(SequentialPipelineBlocks): Outputs: resized_image (`Image`): TODO: Add description. + height (`int`): + The height in pixels of the resized image. + width (`int`): + The width in pixels of the resized image. image_embeds (`Tensor`): The image embeddings """ @@ -101,6 +105,10 @@ class WanFLF2VImageEncoderStep(SequentialPipelineBlocks): Outputs: resized_image (`Image`): TODO: Add description. + height (`int`): + The height in pixels of the resized image. + width (`int`): + The width in pixels of the resized image. resized_last_image (`Image`): TODO: Add description. image_embeds (`Tensor`): @@ -129,18 +137,22 @@ class WanAutoImageEncoderStep(AutoPipelineBlocks): image_processor (`CLIPImageProcessor`) image_encoder (`CLIPVisionModel`) Inputs: - image (`Image`, *optional*): + image (`Image`, *optional*, required, depending on the workflow): TODO: Add description. height (`int`, *optional*, defaults to 480): TODO: Add description. width (`int`, *optional*, defaults to 832): TODO: Add description. - last_image (`Image`, *optional*): + last_image (`Image`, *optional*, required, depending on the workflow): The last frameimage Outputs: resized_image (`Image`): TODO: Add description. + height (`int`): + The height in pixels of the resized image. + width (`int`): + The width in pixels of the resized image. resized_last_image (`Image`): TODO: Add description. image_embeds (`Tensor`): @@ -193,6 +205,10 @@ class WanImage2VideoVaeEncoderStep(SequentialPipelineBlocks): Outputs: resized_image (`Image`): TODO: Add description. + height (`int`): + The height in pixels of the resized image. + width (`int`): + The width in pixels of the resized image. first_frame_latents (`Tensor`): video latent representation with the first frame image condition image_condition_latents (`Tensor | NoneType`): @@ -235,6 +251,10 @@ class WanFLF2VVaeEncoderStep(SequentialPipelineBlocks): Outputs: resized_image (`Image`): TODO: Add description. + height (`int`): + The height in pixels of the resized image. + width (`int`): + The width in pixels of the resized image. resized_last_image (`Image`): TODO: Add description. first_last_frame_latents (`Tensor`): @@ -270,13 +290,13 @@ class WanAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) Inputs: - image (`Image`, *optional*): + image (`Image`, *optional*, required, depending on the workflow): TODO: Add description. height (`int`, *optional*, defaults to 480): TODO: Add description. width (`int`, *optional*, defaults to 832): TODO: Add description. - last_image (`Image`, *optional*): + last_image (`Image`, *optional*, required, depending on the workflow): The last frameimage num_frames (`int`, *optional*, defaults to 81): TODO: Add description. @@ -286,6 +306,10 @@ class WanAutoVaeEncoderStep(AutoPipelineBlocks): Outputs: resized_image (`Image`): TODO: Add description. + height (`int`): + The height in pixels of the resized image. + width (`int`): + The width in pixels of the resized image. resized_last_image (`Image`): TODO: Add description. first_last_frame_latents (`Tensor`): @@ -423,16 +447,16 @@ class WanImage2VideoAutoBlocks(SequentialPipelineBlocks): TODO: Add description. max_sequence_length (`None`, *optional*, defaults to 512): TODO: Add description. - image (`Image`, *optional*): + image (`Image`, *optional*, required, depending on the workflow): TODO: Add description. height (`int`, *optional*, defaults to 480): - TODO: Add description. + The target height of the video. width (`int`, *optional*, defaults to 832): - TODO: Add description. - last_image (`Image`, *optional*): + The target width of the video. + last_image (`Image`, *optional*, required, depending on the workflow): The last frameimage num_frames (`int`, *optional*, defaults to 81): - TODO: Add description. + The number of frames in the generated video. generator (`None`, *optional*): TODO: Add description. num_videos_per_prompt (`None`, *optional*, defaults to 1): @@ -474,6 +498,17 @@ class WanImage2VideoAutoBlocks(SequentialPipelineBlocks): "denoise", "decode", ] + # `image_encoder` and `vae_encoder` default `height`/`width` to 480/832 (the target area, the image sets the + # aspect ratio) and `num_frames` to 81, while the denoise step declares them without a default (it fills them in + # from the latents or the components). Both encoders are auto blocks the walk cannot count on, so the pipeline + # declares the three itself + input_overrides = { + "height": InputParam("height", type_hint=int, default=480, description="The target height of the video."), + "width": InputParam("width", type_hint=int, default=832, description="The target width of the video."), + "num_frames": InputParam( + "num_frames", type_hint=int, default=81, description="The number of frames in the generated video." + ), + } _workflow_map = { "image2video": {"image": True, "prompt": True}, diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py index 1689ed91a59f..f3bd2ee4ce25 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py @@ -14,7 +14,7 @@ from ...utils import logging from ..modular_pipeline import SequentialPipelineBlocks -from ..modular_pipeline_utils import OutputParam +from ..modular_pipeline_utils import InputParam, OutputParam from .before_denoise import ( WanPrepareLatentsStep, WanSetTimestepsStep, @@ -158,7 +158,7 @@ class Wan22VaceBlocks(SequentialPipelineBlocks): width (`None`, *optional*): TODO: Add description. num_frames (`int`, *optional*, defaults to 81): - TODO: Add description. + The number of frames in the generated video. generator (`None`, *optional*): TODO: Add description. num_videos_per_prompt (`None`, *optional*, defaults to 1): @@ -196,6 +196,13 @@ class Wan22VaceBlocks(SequentialPipelineBlocks): "denoise", "decode", ] + # `vace_encoder` defaults `num_frames` to 81 and the denoise step declares it without a default (it fills it in + # from the latents or the components), so the pipeline declares it itself + input_overrides = { + "num_frames": InputParam( + "num_frames", type_hint=int, default=81, description="The number of frames in the generated video." + ) + } @property def description(self): diff --git a/tests/modular_pipelines/test_conditional_pipeline_blocks.py b/tests/modular_pipelines/test_conditional_pipeline_blocks.py index cc0075d0022f..ba967d272709 100644 --- a/tests/modular_pipelines/test_conditional_pipeline_blocks.py +++ b/tests/modular_pipelines/test_conditional_pipeline_blocks.py @@ -21,58 +21,131 @@ OutputParam, SequentialPipelineBlocks, ) -from diffusers.modular_pipelines.modular_pipeline_utils import combine_inputs +from diffusers.modular_pipelines.modular_pipeline_utils import REQUIRED + + +def per_branch(name, record, **kwargs): + """An `InputParam` carrying a per-branch record, the way `ConditionalPipelineBlocks` builds it.""" + return InputParam(name=name, defaults_by_block=record, **kwargs) + + +def conditional_blocks(*named_inputs, default_block_name="block-a"): + """ + A conditional block whose sub-blocks declare the given inputs, e.g. + `conditional_blocks(("block-a", [InputParam("x", default=5)]), ("block-b", [InputParam("x", default=5)]))`. + """ + block_classes = [] + for _, block_inputs in named_inputs: + + class Block(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return list(self._inputs) + + def __call__(self, components, state): + return components, state + + Block._inputs = block_inputs # set after the class statement: a class body cannot see the loop variable + block_classes.append(Block) + + class Conditional(ConditionalPipelineBlocks): + pass + + Conditional.block_classes = block_classes + Conditional.block_names = [name for name, _ in named_inputs] + Conditional.block_trigger_inputs = Conditional.block_names + Conditional.default_block_name = default_block_name + return Conditional() class TestConditionalBlocksInputs: - def test_combine_inputs_agreeing_defaults_stay_untouched(self): - combined = combine_inputs( + def test_agreeing_defaults_stay_untouched(self): + blocks = conditional_blocks( ("block-a", [InputParam(name="x", default=5)]), ("block-b", [InputParam(name="x", default=5)]), ) - assert combined == [InputParam(name="x", default=5)] + assert blocks.inputs == [InputParam(name="x", default=5)] - def test_combine_inputs_first_occurrence_wins(self): - combined = combine_inputs( + def test_first_occurrence_wins(self): + blocks = conditional_blocks( ("block-a", [InputParam(name="x", default=5, description="from block-a")]), ("block-b", [InputParam(name="x", default=5, description="from block-b")]), ) # duplicate inputs keep the first occurrence: block-b's description is dropped - assert combined == [InputParam(name="x", default=5, description="from block-a")] + assert blocks.inputs == [InputParam(name="x", default=5, description="from block-a")] - def test_combine_inputs_none_default_counts_as_disagreement(self): + def test_none_default_counts_as_disagreement(self): # a None default is a sentinel ("user didn't pass this"), it must not be overridden by a sibling's default - combined = combine_inputs( + blocks = conditional_blocks( ("block-a", [InputParam(name="x", default=None)]), ("block-b", [InputParam(name="x", default=5)]), ) - assert combined == [InputParam(name="x", default=None, defaults_by_block={"block-a": None, "block-b": 5})] + assert blocks.inputs == [per_branch("x", {"block-a": None, "block-b": 5})] - def test_combine_inputs_disagreement_records_every_block(self): - combined = combine_inputs( + def test_disagreement_records_every_block(self): + blocks = conditional_blocks( ("block-a", [InputParam(name="x", default=1)]), ("block-b", [InputParam(name="x", default=2)]), ("block-c", [InputParam(name="x", default=1)]), ) # any disagreement records every block's default, including the ones that agree with each other - assert combined == [ - InputParam(name="x", default=None, defaults_by_block={"block-a": 1, "block-b": 2, "block-c": 1}) - ] + assert blocks.inputs == [per_branch("x", {"block-a": 1, "block-b": 2, "block-c": 1})] + + def test_required_everywhere_stays_required(self): + blocks = conditional_blocks( + ("block-a", [InputParam(name="x", required=True)]), + ("block-b", [InputParam(name="x", required=True)]), + ) + assert blocks.inputs == [InputParam(name="x", required=True)] + assert blocks.required_inputs == ["x"] + + def test_required_everywhere_is_optional_when_the_block_can_be_skipped(self): + # no default block: the user can skip the whole block, so nothing it requires is required of the user; the + # record still says which sub-blocks require it + blocks = conditional_blocks( + ("block-a", [InputParam(name="x", required=True)]), + ("block-b", [InputParam(name="x", required=True)]), + default_block_name=None, + ) + assert blocks.inputs == [per_branch("x", {"block-a": REQUIRED, "block-b": REQUIRED})] + assert blocks.required_inputs == [] + + def test_records_a_requirement_other_blocks_do_not_share(self): + # block-b does not declare x at all, so x is not required (only when every block requires it): the record is + # the only place that says block-a requires it + blocks = conditional_blocks( + ("block-a", [InputParam(name="x", required=True)]), + ("block-b", [InputParam(name="y")]), + ) + assert blocks.inputs == [per_branch("x", {"block-a": REQUIRED}), InputParam(name="y")] - def test_combine_inputs_nested_defaults_prefixed(self): + # a requirement against a default is a disagreement like any other + blocks = conditional_blocks( + ("block-a", [InputParam(name="x", required=True)]), + ("block-b", [InputParam(name="x", default=124)]), + ) + assert blocks.inputs == [per_branch("x", {"block-a": REQUIRED, "block-b": 124})] + + def test_nested_defaults_prefixed(self): # block-b is itself a conditional block whose branches already disagree, so its param carries # defaults_by_block; merging at the outer level prefixes those entries with the sub-block name - combined = combine_inputs( + blocks = conditional_blocks( ("block-a", [InputParam(name="x", default=3)]), - ("block-b", [InputParam(name="x", default=None, defaults_by_block={"branch-a": 1, "branch-b": 2})]), + ("block-b", [per_branch("x", {"branch-a": 1, "branch-b": 2})]), ) - assert combined == [ - InputParam( - name="x", - default=None, - defaults_by_block={"block-a": 3, "block-b.branch-a": 1, "block-b.branch-b": 2}, - ) - ] + assert blocks.inputs == [per_branch("x", {"block-a": 3, "block-b.branch-a": 1, "block-b.branch-b": 2})] + + def test_nested_requirements_stay_optional(self): + # both sub-blocks are conditional blocks with one branch requiring x and another silent on it, so each is + # optional with a record; the records agree (REQUIRED, REQUIRED) but x is still not required of the user + blocks = conditional_blocks( + ("block-a", [per_branch("x", {"branch-a": REQUIRED})]), + ("block-b", [per_branch("x", {"branch-b": REQUIRED})]), + ) + assert blocks.inputs == [per_branch("x", {"block-a.branch-a": REQUIRED, "block-b.branch-b": REQUIRED})] + assert blocks.required_inputs == [] class TextToImageBlock(ModularPipelineBlocks): @@ -348,12 +421,7 @@ def test_conflicting_defaults_merge_to_none(self): # and the per-block defaults are recorded in defaults_by_block strength_params = [p for p in AutoImageBlocks().inputs if p.name == "strength"] assert len(strength_params) == 1 - assert strength_params[0] == InputParam( - name="strength", - type_hint=float, - default=None, - defaults_by_block={"inpaint": 0.9999, "img2img": 0.3}, - ) + assert strength_params[0] == per_branch("strength", {"inpaint": 0.9999, "img2img": 0.3}, type_hint=float) # the docstring renders the distinct per-block defaults instead of a single value doc = " ".join(AutoImageBlocks().doc.split()) @@ -384,11 +452,8 @@ def test_standalone_branch_keeps_default(self): def test_nested_defaults_prefixed_with_sub_block_name(self): strength_params = [p for p in NestedImageBlocks().inputs if p.name == "strength"] assert len(strength_params) == 1 - assert strength_params[0] == InputParam( - name="strength", - type_hint=float, - default=None, - defaults_by_block={"refine": 0.9999, "image.inpaint": 0.9999, "image.img2img": 0.3}, + assert strength_params[0] == per_branch( + "strength", {"refine": 0.9999, "image.inpaint": 0.9999, "image.img2img": 0.3}, type_hint=float ) # the docstring renders distinct values only: refine and image.inpaint both use 0.9999, shown once diff --git a/tests/modular_pipelines/test_sequential_pipeline_blocks.py b/tests/modular_pipelines/test_sequential_pipeline_blocks.py index 3da843d0135c..47c17b222a0c 100644 --- a/tests/modular_pipelines/test_sequential_pipeline_blocks.py +++ b/tests/modular_pipelines/test_sequential_pipeline_blocks.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import warnings +import contextlib +import re import pytest @@ -27,6 +28,28 @@ OutputParam, SequentialPipelineBlocks, ) +from diffusers.modular_pipelines.modular_pipeline_utils import REQUIRED +from diffusers.utils import logging +from diffusers.utils.testing_utils import CaptureLogger + + +logger = logging.get_logger("diffusers.modular_pipelines.modular_pipeline") + + +@contextlib.contextmanager +def warns(pattern): + """The body must log a warning matching `pattern`.""" + with CaptureLogger(logger) as logs: + yield + assert re.search(pattern, logs.out), logs.out + + +@contextlib.contextmanager +def quiet(): + """The body must log nothing.""" + with CaptureLogger(logger) as logs: + yield + assert logs.out == "", logs.out class EncodeImageBlock(ModularPipelineBlocks): @@ -89,14 +112,24 @@ def select_block(self, image=None): def strength_auto_blocks(img2img_default, inpaint_default): - """An auto block whose two branches declare `strength` with their own default (`None` for no default).""" + """ + An auto block whose two branches declare `strength` with their own default: `None` for no default, `REQUIRED` for + a branch that requires it, `...` for a branch that does not declare it. + """ + + def strength(default): + if default is ...: + return [] + if default is REQUIRED: + return [InputParam("strength", required=True)] + return [InputParam("strength", default=default)] class Img2ImgBlock(ModularPipelineBlocks): model_name = "test" @property def inputs(self): - return [InputParam("image", required=True), InputParam("strength", default=img2img_default)] + return [InputParam("image", required=True), *strength(img2img_default)] def __call__(self, components, state): return components, state @@ -106,7 +139,7 @@ class InpaintBlock(ModularPipelineBlocks): @property def inputs(self): - return [InputParam("mask", required=True), InputParam("strength", default=inpaint_default)] + return [InputParam("mask", required=True), *strength(inpaint_default)] def __call__(self, components, state): return components, state @@ -119,6 +152,17 @@ class Auto(AutoPipelineBlocks): return Auto +class RequiresStrengthBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("strength", required=True)] + + def __call__(self, components, state): + return components, state + + def required_flags(blocks, *names): return {inp.name: inp.required for inp in blocks.inputs if inp.name in names} @@ -267,6 +311,52 @@ def __call__(self, components, state): assert required_flags(blocks, "image", "image_latents") == {"image": False, "image_latents": False} assert blocks.required_inputs == [] + def test_output_of_some_branches_only_keeps_required_input_listed_but_optional(self): + class EncodeImageAndCountBlock(EncodeImageBlock): + @property + def intermediate_outputs(self): + return [OutputParam("image_latents"), OutputParam("batch_size")] + + class CountBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def intermediate_outputs(self): + return [OutputParam("batch_size")] + + def __call__(self, components, state): + return components, state + + class EncodeOrCountBlocks(ConditionalPipelineBlocks): + block_classes = [EncodeImageAndCountBlock, CountBlock] + block_names = ["encode", "count"] + block_trigger_inputs = ["image"] + default_block_name = "count" + + def select_block(self, image=None): + return "encode" if image is not None else "count" + + class NeedsBothBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("image_latents", required=True), InputParam("batch_size", required=True)] + + def __call__(self, components, state): + return components, state + + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"encode": EncodeOrCountBlocks(), "consume": NeedsBothBlock()} + ) + + # the block always runs: `batch_size` comes out of every branch, so it is produced whatever runs and is not a + # pipeline input. `image_latents` only comes out of `encode`, so it stays listed, optional: pass `image` or + # pass `image_latents` + assert [inp.name for inp in blocks.inputs] == ["image", "image_latents"] + assert required_flags(blocks, "image", "image_latents") == {"image": False, "image_latents": False} + assert blocks.required_inputs == [] + class TestSequentialPipelineBlocksKwargsInputs: def test_kwargs_type_inputs_are_kept_apart(self): @@ -424,27 +514,37 @@ def __call__(self, components, state): return FirstBlock, SecondBlock - def test_conflicting_defaults_are_deprecated_first_wins(self): + def test_two_explicit_values_first_wins(self): + # two explicit values for one shared input: the sequence keeps the first and says so FirstBlock, SecondBlock = self._blocks(0.3, 0.9999) class Blocks(SequentialPipelineBlocks): block_classes = [FirstBlock, SecondBlock] block_names = ["first", "second"] - with pytest.warns(FutureWarning, match="'second' declares default 0.9999 for input 'strength'"): + with warns("'second' declares default 0.9999 for input 'strength'.*keeps the earlier declaration"): inputs = Blocks().inputs - # the first declaration wins until the deprecation turns into an error assert [inp.default for inp in inputs if inp.name == "strength"] == [0.3] - def test_later_none_default_is_not_a_conflict(self): + def test_none_after_a_value_is_not_a_conflict_a_value_after_none_is(self): + # a later block with no default takes whatever is set, nothing to say; a later block with a default that is not + # the one in use is told FirstBlock, SecondBlock = self._blocks(0.3, None) class Blocks(SequentialPipelineBlocks): block_classes = [FirstBlock, SecondBlock] block_names = ["first", "second"] - with warnings.catch_warnings(): - warnings.simplefilter("error", FutureWarning) + with quiet(): + Blocks().inputs + + FirstBlock, SecondBlock = self._blocks(None, 0.3) + + class Blocks(SequentialPipelineBlocks): + block_classes = [FirstBlock, SecondBlock] + block_names = ["first", "second"] + + with warns("'second' declares default 0.3 for input 'strength'"): Blocks().inputs def test_default_override_settles_the_conflict(self): @@ -455,12 +555,11 @@ class Blocks(SequentialPipelineBlocks): block_names = ["first", "second"] input_overrides = {"strength": InputParam("strength", default=0.9999)} - with warnings.catch_warnings(): - warnings.simplefilter("error", FutureWarning) + with quiet(): inputs = Blocks().inputs assert [inp.default for inp in inputs if inp.name == "strength"] == [0.9999] - def test_loop_input_without_default_is_not_a_conflict(self): + def test_loop_input_without_default_is_a_conflict(self): FirstBlock, _ = self._blocks(0.9999, None) class Loop(LoopSequentialPipelineBlocks): @@ -472,13 +571,12 @@ class Loop(LoopSequentialPipelineBlocks): def loop_inputs(self): return [InputParam("timesteps", required=True), InputParam("strength")] - with warnings.catch_warnings(): - warnings.simplefilter("error", FutureWarning) + with warns("'first' declares default 0.9999 for input 'strength'"): inputs = Loop().inputs - # the loop declaration has no opinion, so the step's default is the one the loop reports + # the loop declaration comes first and wins assert [inp.default for inp in inputs if inp.name == "strength"] == [None] - def test_loop_blocks_deprecate_and_accept_the_override(self): + def test_loop_blocks_log_and_accept_the_override(self): FirstBlock, SecondBlock = self._blocks(0.3, 0.9999) class Loop(LoopSequentialPipelineBlocks): @@ -490,75 +588,95 @@ class Loop(LoopSequentialPipelineBlocks): def loop_inputs(self): return [InputParam("timesteps", required=True)] - with pytest.warns(FutureWarning, match="'strength'"): + with warns("'second' declares default 0.9999 for input 'strength'.*The loop keeps the earlier declaration"): Loop().inputs - with warnings.catch_warnings(): - warnings.simplefilter("error", FutureWarning) + with quiet(): inputs = Loop.from_blocks_dict( {"first": FirstBlock(), "second": SecondBlock()}, input_overrides={"strength": InputParam("strength", default=0.9999)}, ).inputs assert [inp.default for inp in inputs if inp.name == "strength"] == [0.9999] - def test_branch_defaults_against_a_value_are_a_conflict(self): + def test_value_after_branch_defaults_is_a_conflict_the_reverse_is_not(self): + # the running branch fills in its default first, so a later leaf default is never the one in use: the composite + # is told. A conditional block after a value declares fallbacks for when the input is still unset, nothing to say FirstBlock, _ = self._blocks(0.5, None) Auto = strength_auto_blocks(0.3, 0.9999) + class AutoThenValue(SequentialPipelineBlocks): + block_classes = [Auto, FirstBlock] + block_names = ["auto", "first"] + + with warns("'first' declares default 0.5 .* declares per-branch defaults"): + AutoThenValue().inputs + class ValueThenAuto(SequentialPipelineBlocks): block_classes = [FirstBlock, Auto] block_names = ["first", "auto"] - with pytest.warns(FutureWarning, match="'auto' declares per-branch defaults .* declares default 0.5"): + with quiet(): inputs = ValueThenAuto().inputs assert [(inp.default, inp.defaults_by_block) for inp in inputs if inp.name == "strength"] == [(0.5, None)] - class AutoThenValue(SequentialPipelineBlocks): - block_classes = [Auto, FirstBlock] - block_names = ["auto", "first"] + def test_declarations_without_a_value_never_conflict(self): + # `None` and per-branch defaults both carry `default=None`: nothing to pre-fill, nothing to disagree about + Auto = strength_auto_blocks(0.3, 0.9999) + NoneBlock, _ = self._blocks(None, None) - with pytest.warns(FutureWarning, match="'first' declares default 0.5 .* declares per-branch defaults"): - AutoThenValue().inputs + class NoneThenAuto(SequentialPipelineBlocks): + block_classes = [NoneBlock, Auto] + block_names = ["first", "auto"] - def test_branch_without_default_has_no_opinion(self): - # one branch agrees with the earlier value and the other declares no default: nothing to settle - FirstBlock, _ = self._blocks(0.3, None) + with quiet(): + (strength,) = [inp for inp in NoneThenAuto().inputs if inp.name == "strength"] + assert (strength.default, strength.defaults_by_block) == (None, None) - class Blocks(SequentialPipelineBlocks): - block_classes = [FirstBlock, strength_auto_blocks(0.3, None)] - block_names = ["first", "auto"] + class AutoThenNone(SequentialPipelineBlocks): + block_classes = [Auto, NoneBlock] + block_names = ["auto", "first"] - with warnings.catch_warnings(): - warnings.simplefilter("error", FutureWarning) - Blocks().inputs + with quiet(): + AutoThenNone().inputs + + class AutoThenAuto(SequentialPipelineBlocks): + block_classes = [Auto, strength_auto_blocks(0.9999, 0.3)] + block_names = ["a", "b"] + + with quiet(): + AutoThenAuto().inputs - def test_branch_defaults_are_carried_onto_an_entry_without_default(self): - FirstBlock, _ = self._blocks(None, None) + def test_value_after_a_required_branch_is_a_conflict(self): + # one branch requires the input and the other does not declare it, so the auto block records + # {"img2img": REQUIRED} with default `None`; the rule compares defaults only, so a later 0.5 is told it is not + # the one in use (it is, in the branch that does not declare the input: a known imprecision of the rule) + FirstBlock, _ = self._blocks(0.5, None) + Auto = strength_auto_blocks(REQUIRED, ...) class Blocks(SequentialPipelineBlocks): - block_classes = [FirstBlock, strength_auto_blocks(0.3, 0.9999)] - block_names = ["first", "auto"] + block_classes = [Auto, FirstBlock] + block_names = ["auto", "first"] - blocks = Blocks() - with warnings.catch_warnings(): - warnings.simplefilter("error", FutureWarning) - (strength,) = [inp for inp in blocks.inputs if inp.name == "strength"] - assert strength.default is None - assert strength.defaults_by_block == {"auto.inpaint": 0.9999, "auto.img2img": 0.3} - (first,) = [inp for inp in blocks.sub_blocks["first"].inputs if inp.name == "strength"] - assert first.defaults_by_block is None + with warns("'first' declares default 0.5 .* per-branch defaults {'img2img': required}"): + (strength,) = [inp for inp in Blocks().inputs if inp.name == "strength"] + assert (strength.required, strength.default, strength.defaults_by_block) == ( + False, + None, + {"img2img": REQUIRED}, + ) - def test_branch_defaults_satisfy_a_later_requirement(self): - class RequiresStrengthBlock(ModularPipelineBlocks): - model_name = "test" + def test_required_branch_does_not_satisfy_a_later_requirement(self): + # the auto block's map holds only a requirement and a `None`, so a later block requiring the input makes it + # a required pipeline input + Auto = strength_auto_blocks(REQUIRED, ...) - @property - def inputs(self): - return [InputParam("strength", required=True)] + class Blocks(SequentialPipelineBlocks): + block_classes = [Auto, RequiresStrengthBlock] + block_names = ["auto", "requires"] - def __call__(self, components, state): - return components, state + assert required_flags(Blocks(), "strength") == {"strength": True} + def test_branch_defaults_satisfy_a_later_requirement(self): class Blocks(SequentialPipelineBlocks): block_classes = [strength_auto_blocks(0.3, 0.9999), RequiresStrengthBlock] block_names = ["auto", "requires"] From 8ca47413ae2d312876022aca20088b4d22b76402 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Fri, 6 Nov 2026 23:18:17 +0100 Subject: [PATCH 03/33] [modular] sync class docstrings with the per-branch default wording Inputs that some branch of a conditional block requires now read "required, depending on the workflow" in the hand-copied class docstrings, matching the renderer. Docstring lines only. Co-Authored-By: Claude Fable 5.1 --- .../anima/modular_blocks_anima.py | 8 ++++---- .../cosmos/modular_blocks_cosmos3_distilled.py | 6 +++--- .../ernie_image/modular_blocks_ernie_image.py | 4 ++-- .../flux/modular_blocks_flux.py | 2 +- .../flux3/modular_blocks_flux3.py | 8 ++++---- .../flux3/modular_blocks_flux3_distilled.py | 6 +++--- .../helios/modular_blocks_helios.py | 12 ++++++------ .../helios/modular_blocks_helios_pyramid.py | 12 ++++++------ .../modular_blocks_helios_pyramid_distilled.py | 12 ++++++------ .../modular_blocks_hunyuan_video1_5.py | 12 ++++++------ .../modular_pipelines/ltx/modular_blocks_ltx.py | 10 +++++----- .../qwenimage/modular_blocks_qwenimage.py | 16 ++++++++-------- .../qwenimage/modular_blocks_qwenimage_edit.py | 2 +- .../qwenimage21/modular_blocks_qwenimage21.py | 10 +++++----- .../modular_blocks_qwenimage21_turbo.py | 10 +++++----- .../modular_blocks_stable_diffusion_3.py | 4 ++-- .../z_image/modular_blocks_z_image.py | 8 ++++---- 17 files changed, 71 insertions(+), 71 deletions(-) diff --git a/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py b/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py index 4de17619f67a..c57fc383decf 100644 --- a/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py +++ b/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py @@ -230,7 +230,7 @@ class AnimaAutoCoreDenoiseStep(AutoPipelineBlocks): Negative T5 prompt attention mask generated by the text encoder step. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. height (`int`, *optional*): The height in pixels of the generated image. @@ -278,7 +278,7 @@ class AnimaAutoVaeImageEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. @@ -330,7 +330,7 @@ class AnimaAutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 512): Maximum sequence length for prompt encoding. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. @@ -340,7 +340,7 @@ class AnimaAutoBlocks(SequentialPipelineBlocks): Torch generator for deterministic generation. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. diff --git a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py index 3d101ffb879a..da0fc79c5e07 100644 --- a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py +++ b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py @@ -34,11 +34,11 @@ class Cosmos3DistilledAutoVaeEncoderStep(ConditionalPipelineBlocks): Latent-frame indexes to preserve from the conditioning video. condition_video_keep (`str`, *optional*, defaults to first): Which end of a longer conditioning video to use: `first` or `last`. - num_frames (`int`, *optional*): + num_frames (`int`, *optional*, required, depending on the workflow): Number of frames to generate. - height (`int`, *optional*): + height (`int`, *optional*, required, depending on the workflow): Height of the generated video in pixels. - width (`int`, *optional*): + width (`int`, *optional*, required, depending on the workflow): Width of the generated video in pixels. image (`None`, *optional*): Reference image for image-to-video conditioning. diff --git a/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py b/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py index 17e4eebaffda..ff81d61aa3c4 100644 --- a/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py +++ b/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py @@ -39,7 +39,7 @@ class ErnieImageAutoPromptEnhancerStep(ConditionalPipelineBlocks): pe (`Ministral3ForCausalLM`) pe_tokenizer (`AutoTokenizer`) Inputs: - prompt (`str`, *optional*): + prompt (`str`, *optional*, required, depending on the workflow): The prompt or prompts to guide image generation. height (`int`, *optional*): The height in pixels of the generated image. @@ -146,7 +146,7 @@ class ErnieImageAutoBlocks(SequentialPipelineBlocks): image_processor (`VaeImageProcessor`) Inputs: - prompt (`str`, *optional*): + prompt (`str`, *optional*, required, depending on the workflow): The prompt or prompts to guide image generation. height (`int`, *optional*): The height in pixels of the generated image. diff --git a/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py b/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py index 464d27c1f278..6a003676c5f2 100644 --- a/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py +++ b/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py @@ -301,7 +301,7 @@ class FluxAutoBeforeDenoiseStep(AutoPipelineBlocks): TODO: Add description. guidance_scale (`None`, *optional*, defaults to 3.5): TODO: Add description. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): The image latents to use for the denoising process. Can be generated in vae encoder and packed in input step. prompt_embeds (`None`, *optional*): diff --git a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py index 0dc01cb5cbc3..204bfaf9bdce 100644 --- a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py +++ b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py @@ -82,7 +82,7 @@ class Flux3AutoVaeEncoderStep(AutoPipelineBlocks): image_processor (`Flux3ImageProcessor`) vae (`AutoencoderKLFlux2`) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): The reference image or images to condition the generation on. One reference is kept whole (padded to the 16-pixel grid, up to 4 megapixels); several are fitted under 1 megapixel each and cropped to the grid @@ -277,7 +277,7 @@ class Flux3AutoCoreDenoiseStep(AutoPipelineBlocks): Pre-generated text embeddings. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): Pre-generated negative text embeddings. Can be generated from text_encoder step. - image_cond_latents (`list`, *optional*): + image_cond_latents (`list`, *optional*, required, depending on the workflow): Normalized packed latents of the reference images, one tensor of shape (B, C, height_i//16, width_i//16) per image height (`int`, *optional*, defaults to None or 1024, depending on the workflow): @@ -354,7 +354,7 @@ class Flux3AutoBlocks(SequentialPipelineBlocks): Inputs: prompt (`str`): The edit instruction to expand into the full caption (one per call). - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): The reference image(s) of the edit, at most 10; the same ones the pipeline receives. boxes (`list`, *optional*): Layout boxes, each {'id', 'desc', 'from', 'src_bbox', 'tgt_bbox'} with the boxes as [y_min, x_min, y_max, @@ -370,7 +370,7 @@ class Flux3AutoBlocks(SequentialPipelineBlocks): Indices of the text-encoder hidden layers whose states are stacked to form the text embeddings num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image_cond_latents (`list`, *optional*): + image_cond_latents (`list`, *optional*, required, depending on the workflow): Normalized packed latents of the reference images, one tensor of shape (B, C, height_i//16, width_i//16) per image height (`int`, *optional*, defaults to None or 1024, depending on the workflow): diff --git a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py index 92e2495dcd26..c68ff5435ee8 100644 --- a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py +++ b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py @@ -197,7 +197,7 @@ class Flux3DistilledAutoCoreDenoiseStep(AutoPipelineBlocks): The number of images to generate per prompt. prompt_embeds (`Tensor`): Pre-generated text embeddings. Can be generated from text_encoder step. - image_cond_latents (`list`, *optional*): + image_cond_latents (`list`, *optional*, required, depending on the workflow): Normalized packed latents of the reference images, one tensor of shape (B, C, height_i//16, width_i//16) per image height (`int`, *optional*, defaults to None or 1024, depending on the workflow): @@ -274,7 +274,7 @@ class Flux3DistilledAutoBlocks(SequentialPipelineBlocks): Inputs: prompt (`str`): The edit instruction to expand into the full caption (one per call). - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): The reference image(s) of the edit, at most 10; the same ones the pipeline receives. boxes (`list`, *optional*): Layout boxes, each {'id', 'desc', 'from', 'src_bbox', 'tgt_bbox'} with the boxes as [y_min, x_min, y_max, @@ -288,7 +288,7 @@ class Flux3DistilledAutoBlocks(SequentialPipelineBlocks): Indices of the text-encoder hidden layers whose states are stacked to form the text embeddings num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image_cond_latents (`list`, *optional*): + image_cond_latents (`list`, *optional*, required, depending on the workflow): Normalized packed latents of the reference images, one tensor of shape (B, C, height_i//16, width_i//16) per image height (`int`, *optional*, defaults to None or 1024, depending on the workflow): diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py index eb0e74c6c2e0..cd1e571bdd79 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py @@ -52,7 +52,7 @@ class HeliosAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) Inputs: - video (`None`, *optional*): + video (`None`, *optional*, required, depending on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -62,7 +62,7 @@ class HeliosAutoVaeEncoderStep(AutoPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. Outputs: @@ -364,7 +364,7 @@ class HeliosAutoCoreDenoiseStep(ConditionalPipelineBlocks): text embeddings used to guide the image generation. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): negative text embeddings used to guide the image generation. Can be generated from text_encoder step. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. @@ -468,7 +468,7 @@ class HeliosAutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 512): Maximum sequence length for prompt encoding. - video (`None`, *optional*): + video (`None`, *optional*, required, depending on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -478,11 +478,11 @@ class HeliosAutoBlocks(SequentialPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. num_videos_per_prompt (`int`, *optional*, defaults to 1): Number of videos to generate per prompt. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py index 7ae868f7e8a7..03abbcb7876d 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py @@ -51,7 +51,7 @@ class HeliosPyramidAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) Inputs: - video (`None`, *optional*): + video (`None`, *optional*, required, depending on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -61,7 +61,7 @@ class HeliosPyramidAutoVaeEncoderStep(AutoPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. Outputs: @@ -350,7 +350,7 @@ class HeliosPyramidAutoCoreDenoiseStep(ConditionalPipelineBlocks): text embeddings used to guide the image generation. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): negative text embeddings used to guide the image generation. Can be generated from text_encoder step. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. @@ -450,7 +450,7 @@ class HeliosPyramidAutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 512): Maximum sequence length for prompt encoding. - video (`None`, *optional*): + video (`None`, *optional*, required, depending on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -460,11 +460,11 @@ class HeliosPyramidAutoBlocks(SequentialPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. num_videos_per_prompt (`int`, *optional*, defaults to 1): Number of videos to generate per prompt. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py index 3bba29ac4485..6790965f4cc0 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py @@ -51,7 +51,7 @@ class HeliosPyramidDistilledAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) Inputs: - video (`None`, *optional*): + video (`None`, *optional*, required, depending on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -61,7 +61,7 @@ class HeliosPyramidDistilledAutoVaeEncoderStep(AutoPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. Outputs: @@ -352,7 +352,7 @@ class HeliosPyramidDistilledAutoCoreDenoiseStep(ConditionalPipelineBlocks): text embeddings used to guide the image generation. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): negative text embeddings used to guide the image generation. Can be generated from text_encoder step. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. @@ -458,7 +458,7 @@ class HeliosPyramidDistilledAutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 512): Maximum sequence length for prompt encoding. - video (`None`, *optional*): + video (`None`, *optional*, required, depending on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -468,11 +468,11 @@ class HeliosPyramidDistilledAutoBlocks(SequentialPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. num_videos_per_prompt (`int`, *optional*, defaults to 1): Number of videos to generate per prompt. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. diff --git a/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py b/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py index 4164a8fed51f..ac40274b0dc5 100644 --- a/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py +++ b/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py @@ -252,7 +252,7 @@ class HunyuanVideo15AutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLHunyuanVideo15`) video_processor (`HunyuanVideo15ImageProcessor`) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. @@ -295,7 +295,7 @@ class HunyuanVideo15AutoImageEncoderStep(AutoPipelineBlocks): image_encoder (`SiglipVisionModel`) feature_extractor (`SiglipImageProcessor`) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. Outputs: @@ -351,7 +351,7 @@ class HunyuanVideo15AutoCoreDenoiseStep(AutoPipelineBlocks): The number of images to generate per prompt. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): Pre-encoded image latents from the VAE encoder step, used as conditioning for I2V. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. @@ -412,7 +412,7 @@ class HunyuanVideo15AutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. @@ -431,7 +431,7 @@ class HunyuanVideo15AutoBlocks(SequentialPipelineBlocks): Pre-generated noisy latents for image generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): Pre-encoded image latents from the VAE encoder step, used as conditioning for I2V. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. @@ -485,7 +485,7 @@ class HunyuanVideo15Image2VideoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. diff --git a/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py b/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py index 70cb18bdad29..42d8cce1f808 100644 --- a/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py +++ b/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py @@ -242,7 +242,7 @@ class LTXAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLLTXVideo`) video_processor (`VideoProcessor`) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -311,7 +311,7 @@ class LTXAutoCoreDenoiseStep(AutoPipelineBlocks): Pre-generated noisy latents for image generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): TODO: Add description. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. @@ -356,7 +356,7 @@ class LTXAutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 128): Maximum sequence length for prompt encoding. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -378,7 +378,7 @@ class LTXAutoBlocks(SequentialPipelineBlocks): TODO: Add description. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): TODO: Add description. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. @@ -433,7 +433,7 @@ class LTXImage2VideoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 128): Maximum sequence length for prompt encoding. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py index cd66eefa7201..a6a19d36b94f 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py @@ -70,7 +70,7 @@ class QwenImageAutoTextEncoderStep(AutoPipelineBlocks): The tokenizer to use guider (`ClassifierFreeGuidance`) Inputs: - prompt (`str`, *optional*): + prompt (`str`, *optional*, required, depending on the workflow): The prompt or prompts to guide image generation. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -220,7 +220,7 @@ class QwenImageOptionalControlNetVaeEncoderStep(AutoPipelineBlocks): (`VaeImageProcessor`) Inputs: - control_image (`Image`, *optional*): + control_image (`Image`, *optional*, required, depending on the workflow): Control image for ControlNet conditioning. height (`int`, *optional*): The height in pixels of the generated image. @@ -1130,15 +1130,15 @@ class QwenImageAutoBlocks(SequentialPipelineBlocks): (`FlowMatchEulerDiscreteScheduler`) transformer (`QwenImageTransformer2DModel`) Inputs: - prompt (`str`, *optional*): + prompt (`str`, *optional*, required, depending on the workflow): The prompt or prompts to guide image generation. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 1024): Maximum sequence length for prompt encoding. - mask_image (`Image`, *optional*): + mask_image (`Image`, *optional*, required, depending on the workflow): Mask image for inpainting. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. @@ -1148,7 +1148,7 @@ class QwenImageAutoBlocks(SequentialPipelineBlocks): Padding for mask cropping in inpainting. generator (`Generator`, *optional*): Torch generator for deterministic generation. - control_image (`Image`, *optional*): + control_image (`Image`, *optional*, required, depending on the workflow): Control image for ControlNet conditioning. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. @@ -1170,13 +1170,13 @@ class QwenImageAutoBlocks(SequentialPipelineBlocks): Additional kwargs for attention processors. **denoiser_input_fields (`None`, *optional*): conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, required, depending on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. processed_mask_image (`Tensor`, *optional*): The processed mask image strength (`float`, *optional*, defaults to 0.9): Strength for img2img/inpainting. - control_image_latents (`Tensor`, *optional*): + control_image_latents (`Tensor`, *optional*, required, depending on the workflow): The control image latents to use for the denoising process. Can be generated in controlnet vae encoder step. control_guidance_start (`float`, *optional*, defaults to 0.0): diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py index e5f8753253f8..689bf8516b0d 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py @@ -736,7 +736,7 @@ class QwenImageEditAutoBlocks(SequentialPipelineBlocks): The prompt or prompts to guide image generation. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. - mask_image (`Image`, *optional*): + mask_image (`Image`, *optional*, required, depending on the workflow): Mask image for inpainting. padding_mask_crop (`int`, *optional*): Padding for mask cropping in inpainting. diff --git a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py index c4c0301604bd..87a63adb513e 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py +++ b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py @@ -108,7 +108,7 @@ class QwenImage21AutoTextEncoderStep(AutoPipelineBlocks): (`Qwen3VLProcessor`) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -208,7 +208,7 @@ class QwenImage21AutoVaeEncoderStep(AutoPipelineBlocks): image_processor (`VaeImageProcessor`) vae (`AutoencoderKLQwenImage21`) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -438,7 +438,7 @@ class QwenImage21AutoCoreDenoiseStep(AutoPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`list`, *optional*): + image_latents (`list`, *optional*, required, depending on the workflow): Normalized latents of each condition image. Can be generated from vae_encoder step. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. @@ -510,7 +510,7 @@ class QwenImage21AutoBlocks(SequentialPipelineBlocks): (`ClassifierFreeGuidance`) transformer (`QwenImage21Transformer2DModel`) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -526,7 +526,7 @@ class QwenImage21AutoBlocks(SequentialPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`list`, *optional*): + image_latents (`list`, *optional*, required, depending on the workflow): Normalized latents of each condition image. Can be generated from vae_encoder step. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. diff --git a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py index 2c975b324cc6..276e471ae962 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py +++ b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py @@ -100,7 +100,7 @@ class QwenImage21TurboAutoTextEncoderStep(AutoPipelineBlocks): (`Qwen3VLProcessor`) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -192,7 +192,7 @@ class QwenImage21TurboAutoVaeEncoderStep(AutoPipelineBlocks): image_processor (`VaeImageProcessor`) vae (`AutoencoderKLQwenImage21`) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -407,7 +407,7 @@ class QwenImage21TurboAutoCoreDenoiseStep(AutoPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`list`, *optional*): + image_latents (`list`, *optional*, required, depending on the workflow): Normalized latents of each condition image. Can be generated from vae_encoder step. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. @@ -483,7 +483,7 @@ class QwenImage21TurboAutoBlocks(SequentialPipelineBlocks): sample_sigmas (default: None): Sampling grid the checkpoint ships with, used when `sigmas` is not passed. Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, required, depending on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -497,7 +497,7 @@ class QwenImage21TurboAutoBlocks(SequentialPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`list`, *optional*): + image_latents (`list`, *optional*, required, depending on the workflow): Normalized latents of each condition image. Can be generated from vae_encoder step. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. diff --git a/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py b/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py index 12d5346387f1..f93abec2cb04 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py @@ -241,7 +241,7 @@ class StableDiffusion3AutoCoreDenoiseStep(AutoPipelineBlocks): The height in pixels of the generated image. width (`None`, *optional*): The width in pixels of the generated image. - image_latents (`None`, *optional*): + image_latents (`None`, *optional*, required, depending on the workflow): Latent input image_latents to be processed. latents (`Tensor | NoneType`, *optional*): Pre-generated noisy latents to be used as inputs for image generation. @@ -330,7 +330,7 @@ class StableDiffusion3AutoBlocks(SequentialPipelineBlocks): One or a list of torch generator(s) to make generation deterministic. num_images_per_prompt (`None`, *optional*, defaults to 1): The number of images to generate per prompt. - image_latents (`None`, *optional*): + image_latents (`None`, *optional*, required, depending on the workflow): Latent input image_latents to be processed. latents (`Tensor | NoneType`, *optional*): Pre-generated noisy latents to be used as inputs for image generation. diff --git a/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py b/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py index a0bf2e3a8cc3..66565b260262 100644 --- a/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py +++ b/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py @@ -188,7 +188,7 @@ class ZImageAutoDenoiseStep(AutoPipelineBlocks): TODO: Add description. width (`None`, *optional*): TODO: Add description. - image_latents (`None`, *optional*): + image_latents (`None`, *optional*, required, depending on the workflow): TODO: Add description. latents (`Tensor | NoneType`, *optional*): TODO: Add description. @@ -236,7 +236,7 @@ class ZImageAutoVaeImageEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKL`) image_processor (`VaeImageProcessor`) Inputs: - image (`Image`, *optional*): + image (`Image`, *optional*, required, depending on the workflow): TODO: Add description. height (`None`, *optional*): TODO: Add description. @@ -283,7 +283,7 @@ class ZImageAutoBlocks(SequentialPipelineBlocks): TODO: Add description. max_sequence_length (`None`, *optional*, defaults to 512): TODO: Add description. - image (`Image`, *optional*): + image (`Image`, *optional*, required, depending on the workflow): TODO: Add description. height (`None`, *optional*): TODO: Add description. @@ -293,7 +293,7 @@ class ZImageAutoBlocks(SequentialPipelineBlocks): TODO: Add description. num_images_per_prompt (`None`, *optional*, defaults to 1): TODO: Add description. - image_latents (`None`, *optional*): + image_latents (`None`, *optional*, required, depending on the workflow): TODO: Add description. latents (`Tensor | NoneType`, *optional*): TODO: Add description. From 3bedb58e760086843d302e104e19147ee78cac40 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sun, 8 Nov 2026 10:46:53 +0100 Subject: [PATCH 04/33] [modular] docstring says "depends on the workflow" for a per-branch input An input that a conditional block's branches declare differently rendered as "required or defaults to X, depending on the workflow". The merged walk cannot know which branch runs, nor whether an earlier block produces the value in that workflow, so the line now says only that it depends on the workflow. `get_workflow(name).inputs` gives the exact signature of one workflow; a per-workflow docstring is #15208. The sequence prefixes a conditional sub-block's `defaults_by_block` keys with the sub-block name, so the record says which block's branch each entry is. Co-Authored-By: Claude Fable 5.1 --- .../modular_pipelines/modular_pipeline.py | 19 +++++-- .../modular_pipeline_utils.py | 21 +++----- .../test_conditional_pipeline_blocks.py | 7 ++- .../test_sequential_pipeline_blocks.py | 51 ++++++++++++++++++- 4 files changed, 73 insertions(+), 25 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 36c4333ba89d..e011e64fa5f3 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -782,7 +782,7 @@ def _get_inputs(self): # itself can be skipped): the merged input is optional with default `None`, and `defaults_by_block` records # what each declaring sub-block says. At runtime the sub-block that runs fills in its own default, or raises when # it requires the input and it is missing (see `ModularPipelineBlocks.get_block_state`); - # the docstrings render the defaults, e.g. "defaults to 0.3 or 0.9999, depending on the workflow" + # the docstrings say "depends on the workflow" declarations = {} # declarations: input_name -> {block_name: InputParam}, `None` when the sub-block does not declare the input @@ -1300,10 +1300,19 @@ def _get_inputs(self): if key not in inputs: # otherwise, declare it as an input if not already declared inputs[key] = copy(inp) - if inp.required and key in maybe_outputs: - # the block requires it, but an earlier conditional block can produce it, so change it to optional - # e.g. `prepare_latents` lists `image_latents` as a required input, but `image_encoder` (an conditional block) produces it when `image` - # is passed, so `image_latents` is an optional input for the pipeline, i.e. user can skip it if they pass `image` and run the image_encoder block. + if inp.defaults_by_block: + # a conditional sub-block's record is keyed by its branches; prefix them with the sub-block's name + # so the record says which block's branch each entry is, like a nested conditional block does + # e.g. {"img2img": REQUIRED} on `denoise` -> {"denoise.img2img": REQUIRED} + inputs[key].defaults_by_block = { + f"{block_name}.{k}": v for k, v in inp.defaults_by_block.items() + } + if key in maybe_outputs: + # an earlier conditional block can produce it, so the caller does not have to pass it even if + # this block requires it. + # e.g. `prepare_latents` lists `image_latents` as a required input, but `image_encoder` (a + # conditional block) produces it when `image` is passed, so `image_latents` is an optional input + # for the pipeline: the user can skip it if they pass `image` and run the image_encoder block. inputs[key].required = False continue # if the input is already declared: a later declaration can only make it required, or warn on a conflict in the defaults diff --git a/src/diffusers/modular_pipelines/modular_pipeline_utils.py b/src/diffusers/modular_pipelines/modular_pipeline_utils.py index 5e495c51404c..f9e09afa48e9 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline_utils.py +++ b/src/diffusers/modular_pipelines/modular_pipeline_utils.py @@ -749,14 +749,11 @@ def wrap_text(text, indent, max_length): if not param.required: param_str += ", *optional*" if param.defaults_by_block: - # "required" first when a branch requires it, then one entry per distinct default: - # e.g. {"a": None, "b": 189, "c": 189} -> ", defaults to None or 189, depending on the workflow" - # {"t2v": 124, "ref2v": REQUIRED} -> ", required or defaults to 124, depending on the workflow" - distinct = list(dict.fromkeys(param.defaults_by_block.values())) - parts = ["required"] if REQUIRED in distinct else [] - if any(v is not REQUIRED for v in distinct): - parts.append(f"defaults to {' or '.join(str(v) for v in distinct if v is not REQUIRED)}") - param_str += f", {' or '.join(parts)}, depending on the workflow" + # the branches of a conditional sub-block declare it differently (a default each, or one requires + # it). Which one applies, and whether an earlier block produces the value instead, depends on the + # branch that runs, so the merged docstring says only that. `get_workflow(name).inputs` has the + # exact signature of one workflow + param_str += ", depends on the workflow" elif param.default is not None: param_str += f", defaults to {param.default}" param_str += "):" @@ -832,12 +829,8 @@ def get_type_str(type_hint): if hasattr(param, "required") and not param.required: param_str += ", *optional*" if param.defaults_by_block: - # e.g. {"t2v": 124, "ref2v": REQUIRED} -> ", required or defaults to `124`, depending on the workflow" - distinct = list(dict.fromkeys(param.defaults_by_block.values())) - parts = ["required"] if REQUIRED in distinct else [] - if any(v is not REQUIRED for v in distinct): - parts.append(f"defaults to {' or '.join(f'`{v}`' for v in distinct if v is not REQUIRED)}") - param_str += f", {' or '.join(parts)}, depending on the workflow" + # see `format_input_params` + param_str += ", depends on the workflow" elif param.default is not None: param_str += f", defaults to `{param.default}`" param_str += ")" diff --git a/tests/modular_pipelines/test_conditional_pipeline_blocks.py b/tests/modular_pipelines/test_conditional_pipeline_blocks.py index ba967d272709..34f3e60e6948 100644 --- a/tests/modular_pipelines/test_conditional_pipeline_blocks.py +++ b/tests/modular_pipelines/test_conditional_pipeline_blocks.py @@ -423,9 +423,9 @@ def test_conflicting_defaults_merge_to_none(self): assert len(strength_params) == 1 assert strength_params[0] == per_branch("strength", {"inpaint": 0.9999, "img2img": 0.3}, type_hint=float) - # the docstring renders the distinct per-block defaults instead of a single value + # the docstring does not pick one of the per-block defaults doc = " ".join(AutoImageBlocks().doc.split()) - assert "strength (`float`, *optional*, defaults to 0.9999 or 0.3, depending on the workflow):" in doc + assert "strength (`float`, *optional*, depends on the workflow):" in doc def test_branch_resolves_own_default(self): pipe = AutoImageBlocks().init_pipeline() @@ -456,9 +456,8 @@ def test_nested_defaults_prefixed_with_sub_block_name(self): "strength", {"refine": 0.9999, "image.inpaint": 0.9999, "image.img2img": 0.3}, type_hint=float ) - # the docstring renders distinct values only: refine and image.inpaint both use 0.9999, shown once doc = " ".join(NestedImageBlocks().doc.split()) - assert "strength (`float`, *optional*, defaults to 0.9999 or 0.3, depending on the workflow):" in doc + assert "strength (`float`, *optional*, depends on the workflow):" in doc def test_explicit_none_reaches_every_branch(self): # a passed None is a value, not "use the default": the block keeps it instead of filling in its default diff --git a/tests/modular_pipelines/test_sequential_pipeline_blocks.py b/tests/modular_pipelines/test_sequential_pipeline_blocks.py index 47c17b222a0c..e061bedf7737 100644 --- a/tests/modular_pipelines/test_sequential_pipeline_blocks.py +++ b/tests/modular_pipelines/test_sequential_pipeline_blocks.py @@ -357,6 +357,53 @@ def __call__(self, components, state): assert required_flags(blocks, "image", "image_latents") == {"image": False, "image_latents": False} assert blocks.required_inputs == [] + def test_record_of_a_conditional_sub_block_is_prefixed_with_its_name(self): + # `denoise` is a conditional block whose img2img branch requires `image_latents`, recorded as + # {"img2img": REQUIRED}; the sequence keeps the record, keyed by `denoise.img2img`, and the input optional: + # `encode` may produce it, so whether the caller has to pass it depends on the workflow + class Img2ImgBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("image_latents", required=True), InputParam("strength", default=0.3)] + + def __call__(self, components, state): + return components, state + + class Text2ImgBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("strength", default=0.9999)] + + def __call__(self, components, state): + return components, state + + class DenoiseBlocks(ConditionalPipelineBlocks): + block_classes = [Img2ImgBlock, Text2ImgBlock] + block_names = ["img2img", "text2img"] + block_trigger_inputs = ["image_latents"] + default_block_name = "text2img" + + def select_block(self, image_latents=None): + return "img2img" if image_latents is not None else "text2img" + + class Blocks(SequentialPipelineBlocks): + block_classes = [SkippableEncodeBlocks, DenoiseBlocks] + block_names = ["encode", "denoise"] + + inputs = {inp.name: inp for inp in Blocks().inputs} + assert (inputs["image_latents"].required, inputs["image_latents"].defaults_by_block) == ( + False, + {"denoise.img2img": REQUIRED}, + ) + assert inputs["strength"].defaults_by_block == {"denoise.img2img": 0.3, "denoise.text2img": 0.9999} + # the conditional block on its own keeps the branch names + inputs = {inp.name: inp for inp in DenoiseBlocks().inputs} + assert inputs["image_latents"].defaults_by_block == {"img2img": REQUIRED} + class TestSequentialPipelineBlocksKwargsInputs: def test_kwargs_type_inputs_are_kept_apart(self): @@ -657,12 +704,12 @@ class Blocks(SequentialPipelineBlocks): block_classes = [Auto, FirstBlock] block_names = ["auto", "first"] - with warns("'first' declares default 0.5 .* per-branch defaults {'img2img': required}"): + with warns("'first' declares default 0.5 .* per-branch defaults {'auto.img2img': required}"): (strength,) = [inp for inp in Blocks().inputs if inp.name == "strength"] assert (strength.required, strength.default, strength.defaults_by_block) == ( False, None, - {"img2img": REQUIRED}, + {"auto.img2img": REQUIRED}, ) def test_required_branch_does_not_satisfy_a_later_requirement(self): From 3638fd19908498c23c6a503617cf0eca8a9555a3 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sun, 8 Nov 2026 10:46:53 +0100 Subject: [PATCH 05/33] [modular] P-H3 Image: declare height and width on the pipeline The resize step reads them with no default and the prompt upsampler defaults them to 2912, which the sequence now warns about. The pipeline declares both itself, optional with no default, so each workflow keeps its own. Co-Authored-By: Claude Fable 5.1 --- .../p_h3_image/modular_blocks_p_h3_image.py | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py b/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py index 020201616522..8baad8a3983e 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py +++ b/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py @@ -15,7 +15,7 @@ import torch from ..modular_pipeline import AutoPipelineBlocks, SequentialPipelineBlocks -from ..modular_pipeline_utils import OutputParam +from ..modular_pipeline_utils import InputParam, OutputParam from .before_denoise import ( PH3ImageEditPrepareLatentsStep, PH3ImageEditPrepareLayoutStep, @@ -47,7 +47,7 @@ class PH3ImageAutoBeforeEncodeStep(AutoPipelineBlocks): 4194304) vision_pixels (default: 1000000) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, depends on the workflow): The image to edit, or a list of at most 8 images: the first is edited, the others are references. height (`int`, *optional*): Height of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for `text2image`, @@ -97,7 +97,7 @@ class PH3ImageAutoTextEncoderStep(AutoPipelineBlocks): Inputs: prompt (`str`): The prompt, or the edit instruction when `image` is passed. A single string. - vision_images (`list`, *optional*): + vision_images (`list`, *optional*, depends on the workflow): The images of the edit resized for the text encoder, the image to edit first. Outputs: @@ -132,7 +132,7 @@ class PH3ImageAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLPH3Image`) Inputs: - reference_images (`list`, *optional*): + reference_images (`list`, *optional*, depends on the workflow): The images of the edit resized for the VAE, the image to edit first. Outputs: @@ -329,17 +329,17 @@ class PH3ImageAutoDenoiseStep(AutoPipelineBlocks): Inputs: text_token_tags (`Tensor`): The per-row modality tag of every row of `prompt_embeds`. - condition_latents (`list`, *optional*): + condition_latents (`list`, *optional*, depends on the workflow): The conditioning latents of the images of the edit. Their shapes give the geometry of the reference blocks. - height (`int`, *optional*, defaults to None or 2912, depending on the workflow): + height (`int`, *optional*, depends on the workflow): Height of the edited image in pixels. - width (`int`, *optional*, defaults to None or 2912, depending on the workflow): + width (`int`, *optional*, depends on the workflow): Width of the edited image in pixels. generator (`Generator`, *optional*): The generator of the request. The conditioning noise is drawn from it first, one draw per condition, before the noise of the generated rows. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated image noise of shape `(1, latent_channels, 1, latent_height, latent_width)`, used instead of the draw. sigmas (`list`, *optional*): @@ -397,7 +397,7 @@ class PH3ImageBlocks(SequentialPipelineBlocks): passed. Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, depends on the workflow): The image to edit, or a list of at most 8 images: the first is edited, the others are references. height (`int`, *optional*): Height of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for `text2image`, @@ -409,19 +409,19 @@ class PH3ImageBlocks(SequentialPipelineBlocks): `canvas_min_pixels`, and works best at 4 or 8 megapixels. prompt (`str`): The prompt, or the edit instruction when `image` is passed. A single string. - max_new_tokens (`int`, *optional*, defaults to 1536 or 1024, depending on the workflow): + max_new_tokens (`int`, *optional*, depends on the workflow): The most tokens the upsampler may generate, its reasoning included. - vision_images (`list`, *optional*): + vision_images (`list`, *optional*, depends on the workflow): The images of the edit resized for the text encoder, the image to edit first. - reference_images (`list`, *optional*): + reference_images (`list`, *optional*, depends on the workflow): The images of the edit resized for the VAE, the image to edit first. - condition_latents (`list`, *optional*): + condition_latents (`list`, *optional*, depends on the workflow): The conditioning latents of the images of the edit. Their shapes give the geometry of the reference blocks. generator (`Generator`, *optional*): The generator of the request. The conditioning noise is drawn from it first, one draw per condition, before the noise of the generated rows. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated image noise of shape `(1, latent_channels, 1, latent_height, latent_width)`, used instead of the draw. sigmas (`list`, *optional*): @@ -450,6 +450,27 @@ class PH3ImageBlocks(SequentialPipelineBlocks): PH3ImageDecodeStep, ] block_names = ["before_encode", "prompt_upsampler", "text_encoder", "vae_encoder", "denoise", "decode"] + # `before_encode` reads `height`/`width` with no default (an edit derives them from the image) and the prompt + # upsampler defaults them to 2912, so the pipeline declares them itself: optional, no default, each workflow keeps + # its own + input_overrides = { + "height": InputParam.template( + "height", + description=( + "Height of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for " + "`text2image`, which works best at 8 megapixels. An edit defaults to the first image's size, at " + "least `canvas_min_pixels`, and works best at 4 or 8 megapixels." + ), + ), + "width": InputParam.template( + "width", + description=( + "Width of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for " + "`text2image`, which works best at 8 megapixels. An edit defaults to the first image's size, at " + "least `canvas_min_pixels`, and works best at 4 or 8 megapixels." + ), + ), + } _workflow_map = { "text2image": {"prompt": True}, "image_conditioned": {"prompt": True, "image": True}, From dc5c91bd227ab9f7409bc5bb7c12febc1bf58bbb Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sun, 8 Nov 2026 10:46:53 +0100 Subject: [PATCH 06/33] [modular] regenerate auto docstrings for the workflow-dependent wording Co-Authored-By: Claude Fable 5.1 --- .../anima/modular_blocks_anima.py | 8 +- .../cosmos/modular_blocks_cosmos3.py | 52 ++++++------ .../modular_blocks_cosmos3_distilled.py | 6 +- .../ernie_image/modular_blocks_ernie_image.py | 4 +- .../flux/modular_blocks_flux.py | 2 +- .../flux3/modular_blocks_flux3.py | 16 ++-- .../flux3/modular_blocks_flux3_distilled.py | 14 ++-- .../helios/modular_blocks_helios.py | 12 +-- .../helios/modular_blocks_helios_pyramid.py | 12 +-- ...modular_blocks_helios_pyramid_distilled.py | 12 +-- .../modular_blocks_hunyuan_video1_5.py | 12 +-- .../ltx/modular_blocks_ltx.py | 10 +-- .../ltx2/modular_blocks_ltx2.py | 80 +++++++++---------- .../ltx2/modular_blocks_ltx25.py | 22 ++--- .../minimax_h3/modular_blocks_minimax_h3.py | 30 +++---- .../p_h3_image/prompt_upsampler.py | 4 +- .../modular_blocks_pika_h3_flex.py | 40 +++++----- .../qwenimage/modular_blocks_qwenimage.py | 16 ++-- .../modular_blocks_qwenimage_edit.py | 2 +- .../qwenimage21/modular_blocks_qwenimage21.py | 10 +-- .../modular_blocks_qwenimage21_turbo.py | 10 +-- .../modular_blocks_stable_diffusion_3.py | 4 +- .../modular_blocks_stable_diffusion_xl.py | 20 ++--- .../wan/modular_blocks_wan_i2v.py | 12 +-- .../z_image/modular_blocks_z_image.py | 8 +- 25 files changed, 209 insertions(+), 209 deletions(-) diff --git a/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py b/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py index c57fc383decf..efeb99e49b08 100644 --- a/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py +++ b/src/diffusers/modular_pipelines/anima/modular_blocks_anima.py @@ -230,7 +230,7 @@ class AnimaAutoCoreDenoiseStep(AutoPipelineBlocks): Negative T5 prompt attention mask generated by the text encoder step. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. height (`int`, *optional*): The height in pixels of the generated image. @@ -278,7 +278,7 @@ class AnimaAutoVaeImageEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLQwenImage`) image_processor (`VaeImageProcessor`) Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. @@ -330,7 +330,7 @@ class AnimaAutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 512): Maximum sequence length for prompt encoding. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. @@ -340,7 +340,7 @@ class AnimaAutoBlocks(SequentialPipelineBlocks): Torch generator for deterministic generation. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. diff --git a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py index b1861c86c96c..436d82d73ea7 100644 --- a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py +++ b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3.py @@ -127,7 +127,7 @@ class Cosmos3AutoTextEncoderStep(AutoPipelineBlocks): default_use_system_prompt (default: True) enable_safety_checker (default: True) Inputs: - control_videos (`dict`, *optional*, required, depending on the workflow): + control_videos (`dict`, *optional*, depends on the workflow): Mapping of hint name (edge/blur/depth/seg/wsm) to the control video for that modality. height (`int`, *optional*): Height of the generated video in pixels. @@ -143,9 +143,9 @@ class Cosmos3AutoTextEncoderStep(AutoPipelineBlocks): The text prompt that guides Cosmos3 generation. negative_prompt (`str`, *optional*): The negative text prompt used for classifier-free guidance. - use_system_prompt (`bool`, *optional*, defaults to True or None, depending on the workflow): + use_system_prompt (`bool`, *optional*, depends on the workflow): Whether to prepend the Cosmos3 transfer system prompt. - action (`CosmosActionCondition`, *optional*, required, depending on the workflow): + action (`CosmosActionCondition`, *optional*, depends on the workflow): Action-conditioning metadata and its reference visual input. fps (`float`, *optional*, defaults to 24.0): Frame rate of the generated video. @@ -207,7 +207,7 @@ class Cosmos3AutoVaeEncoderStep(ConditionalPipelineBlocks): vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) Inputs: - action (`CosmosActionCondition`, *optional*, required, depending on the workflow): + action (`CosmosActionCondition`, *optional*, depends on the workflow): Action-conditioning metadata and its reference visual input. video (`None`, *optional*): Reference video for video-to-video conditioning. @@ -215,11 +215,11 @@ class Cosmos3AutoVaeEncoderStep(ConditionalPipelineBlocks): Latent-frame indexes to preserve from the conditioning video. condition_video_keep (`str`, *optional*, defaults to first): Which end of a longer conditioning video to use: `first` or `last`. - num_frames (`int`, *optional*, required, depending on the workflow): + num_frames (`int`, *optional*, depends on the workflow): Number of frames to generate. - height (`int`, *optional*, required, depending on the workflow): + height (`int`, *optional*, depends on the workflow): Height of the generated video in pixels. - width (`int`, *optional*, required, depending on the workflow): + width (`int`, *optional*, depends on the workflow): Width of the generated video in pixels. image (`None`, *optional*): Reference image for image-to-video conditioning. @@ -283,7 +283,7 @@ class Cosmos3AutoSoundDecodeStep(AutoPipelineBlocks): sound_tokenizer (`Cosmos3AVAEAudioTokenizer`) Inputs: - sound_latents (`Tensor`, *optional*, required, depending on the workflow): + sound_latents (`Tensor`, *optional*, depends on the workflow): Denoised sound latents to decode. Outputs: @@ -320,7 +320,7 @@ class Cosmos3DecodeStep(SequentialPipelineBlocks): Denoised vision latents to decode. output_type (`str`, *optional*, defaults to pil): Output format: 'pil', 'np', 'pt'. - sound_latents (`Tensor`, *optional*, required, depending on the workflow): + sound_latents (`Tensor`, *optional*, depends on the workflow): Denoised sound latents to decode. Outputs: @@ -1046,13 +1046,13 @@ class Cosmos3AutoCoreDenoiseStep(ConditionalPipelineBlocks): Index of the current chunk. previous_output (`None`, *optional*): Decoded pixels of the previous chunk, used to seed later chunks. - control_frames (`dict`, *optional*, required, depending on the workflow): + control_frames (`dict`, *optional*, depends on the workflow): Preprocessed, time-padded control maps in canonical hint order. - chunk_frames (`int`, *optional*, required, depending on the workflow): + chunk_frames (`int`, *optional*, depends on the workflow): Pixel frames per chunk. - total_frames (`int`, *optional*, required, depending on the workflow): + total_frames (`int`, *optional*, depends on the workflow): Total number of output frames. - stride (`int`, *optional*, required, depending on the workflow): + stride (`int`, *optional*, depends on the workflow): Frame stride between chunks. height (`int`): Height of the generated video in pixels. @@ -1089,21 +1089,21 @@ class Cosmos3AutoCoreDenoiseStep(ConditionalPipelineBlocks): Timestep interval [lo, hi] over which text guidance is active (None = always). control_guidance_interval (`tuple`, *optional*): Timestep interval [lo, hi] over which control guidance is active (None = always). - output_chunks (`list`, *optional*, required, depending on the workflow): + output_chunks (`list`, *optional*, depends on the workflow): Decoded pixel chunks accumulated so far. - num_chunks (`int`, *optional*, required, depending on the workflow): + num_chunks (`int`, *optional*, depends on the workflow): Number of autoregressive chunks. x0_tokens_vision (`Tensor`, *optional*): Vision latents encoded from the conditioning image or video. vision_condition_frames (`list`, *optional*): Latent-frame indexes fixed by visual conditioning. - num_frames (`int`, *optional*, required, depending on the workflow): + num_frames (`int`, *optional*, depends on the workflow): Number of frames to generate. latents (`Tensor`, *optional*): Pre-generated noisy vision latents. sound_latents (`Tensor`, *optional*): Pre-generated noisy sound latents. - action (`CosmosActionCondition`, *optional*, required, depending on the workflow): + action (`CosmosActionCondition`, *optional*, depends on the workflow): Action-conditioning metadata. action_condition_frame_indexes (`list`, *optional*): Action-frame indexes fixed by action conditioning. @@ -1238,7 +1238,7 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): (default: False) Inputs: - control_videos (`dict`, *optional*, required, depending on the workflow): + control_videos (`dict`, *optional*, depends on the workflow): Mapping of hint name (edge/blur/depth/seg/wsm) to the control video for that modality. height (`int`, *optional*): Height of the generated video in pixels. @@ -1254,9 +1254,9 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): The text prompt that guides Cosmos3 generation. negative_prompt (`str`, *optional*): The negative text prompt used for classifier-free guidance. - use_system_prompt (`bool`, *optional*, defaults to True or None, depending on the workflow): + use_system_prompt (`bool`, *optional*, depends on the workflow): Whether to prepend the Cosmos3 transfer system prompt. - action (`CosmosActionCondition`, *optional*, required, depending on the workflow): + action (`CosmosActionCondition`, *optional*, depends on the workflow): Action-conditioning metadata and its reference visual input. fps (`float`, *optional*, defaults to 24.0): Frame rate of the generated video. @@ -1276,13 +1276,13 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): Index of the current chunk. previous_output (`None`, *optional*): Decoded pixels of the previous chunk, used to seed later chunks. - control_frames (`dict`, *optional*, required, depending on the workflow): + control_frames (`dict`, *optional*, depends on the workflow): Preprocessed, time-padded control maps in canonical hint order. - chunk_frames (`int`, *optional*, required, depending on the workflow): + chunk_frames (`int`, *optional*, depends on the workflow): Pixel frames per chunk. - total_frames (`int`, *optional*, required, depending on the workflow): + total_frames (`int`, *optional*, depends on the workflow): Total number of output frames. - stride (`int`, *optional*, required, depending on the workflow): + stride (`int`, *optional*, depends on the workflow): Frame stride between chunks. num_first_chunk_conditional_frames (`int`, *optional*, defaults to 0): Number of frames the first chunk reuses from the input video. @@ -1309,9 +1309,9 @@ class Cosmos3OmniBlocks(SequentialPipelineBlocks): Timestep interval [lo, hi] over which text guidance is active (None = always). control_guidance_interval (`tuple`, *optional*): Timestep interval [lo, hi] over which control guidance is active (None = always). - output_chunks (`list`, *optional*, required, depending on the workflow): + output_chunks (`list`, *optional*, depends on the workflow): Decoded pixel chunks accumulated so far. - num_chunks (`int`, *optional*, required, depending on the workflow): + num_chunks (`int`, *optional*, depends on the workflow): Number of autoregressive chunks. x0_tokens_vision (`Tensor`, *optional*): Vision latents encoded from the conditioning image or video. diff --git a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py index da0fc79c5e07..e33f4e4d17a9 100644 --- a/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py +++ b/src/diffusers/modular_pipelines/cosmos/modular_blocks_cosmos3_distilled.py @@ -34,11 +34,11 @@ class Cosmos3DistilledAutoVaeEncoderStep(ConditionalPipelineBlocks): Latent-frame indexes to preserve from the conditioning video. condition_video_keep (`str`, *optional*, defaults to first): Which end of a longer conditioning video to use: `first` or `last`. - num_frames (`int`, *optional*, required, depending on the workflow): + num_frames (`int`, *optional*, depends on the workflow): Number of frames to generate. - height (`int`, *optional*, required, depending on the workflow): + height (`int`, *optional*, depends on the workflow): Height of the generated video in pixels. - width (`int`, *optional*, required, depending on the workflow): + width (`int`, *optional*, depends on the workflow): Width of the generated video in pixels. image (`None`, *optional*): Reference image for image-to-video conditioning. diff --git a/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py b/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py index ff81d61aa3c4..e32b3dbe183e 100644 --- a/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py +++ b/src/diffusers/modular_pipelines/ernie_image/modular_blocks_ernie_image.py @@ -39,7 +39,7 @@ class ErnieImageAutoPromptEnhancerStep(ConditionalPipelineBlocks): pe (`Ministral3ForCausalLM`) pe_tokenizer (`AutoTokenizer`) Inputs: - prompt (`str`, *optional*, required, depending on the workflow): + prompt (`str`, *optional*, depends on the workflow): The prompt or prompts to guide image generation. height (`int`, *optional*): The height in pixels of the generated image. @@ -146,7 +146,7 @@ class ErnieImageAutoBlocks(SequentialPipelineBlocks): image_processor (`VaeImageProcessor`) Inputs: - prompt (`str`, *optional*, required, depending on the workflow): + prompt (`str`, *optional*, depends on the workflow): The prompt or prompts to guide image generation. height (`int`, *optional*): The height in pixels of the generated image. diff --git a/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py b/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py index 6a003676c5f2..64b9372e880b 100644 --- a/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py +++ b/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py @@ -301,7 +301,7 @@ class FluxAutoBeforeDenoiseStep(AutoPipelineBlocks): TODO: Add description. guidance_scale (`None`, *optional*, defaults to 3.5): TODO: Add description. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): The image latents to use for the denoising process. Can be generated in vae encoder and packed in input step. prompt_embeds (`None`, *optional*): diff --git a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py index 204bfaf9bdce..05dfc96be5dc 100644 --- a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py +++ b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3.py @@ -82,7 +82,7 @@ class Flux3AutoVaeEncoderStep(AutoPipelineBlocks): image_processor (`Flux3ImageProcessor`) vae (`AutoencoderKLFlux2`) Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): The reference image or images to condition the generation on. One reference is kept whole (padded to the 16-pixel grid, up to 4 megapixels); several are fitted under 1 megapixel each and cropped to the grid @@ -277,12 +277,12 @@ class Flux3AutoCoreDenoiseStep(AutoPipelineBlocks): Pre-generated text embeddings. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): Pre-generated negative text embeddings. Can be generated from text_encoder step. - image_cond_latents (`list`, *optional*, required, depending on the workflow): + image_cond_latents (`list`, *optional*, depends on the workflow): Normalized packed latents of the reference images, one tensor of shape (B, C, height_i//16, width_i//16) per image - height (`int`, *optional*, defaults to None or 1024, depending on the workflow): + height (`int`, *optional*, depends on the workflow): The height in pixels of the generated image; defaults to the first reference image's own height - width (`int`, *optional*, defaults to None or 1024, depending on the workflow): + width (`int`, *optional*, depends on the workflow): The width in pixels of the generated image; defaults to the first reference image's own width reference_crop (`list | NoneType`, *optional*): Per reference, the box of the reference inside its padded canvas or None, from the preprocess step @@ -354,7 +354,7 @@ class Flux3AutoBlocks(SequentialPipelineBlocks): Inputs: prompt (`str`): The edit instruction to expand into the full caption (one per call). - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): The reference image(s) of the edit, at most 10; the same ones the pipeline receives. boxes (`list`, *optional*): Layout boxes, each {'id', 'desc', 'from', 'src_bbox', 'tgt_bbox'} with the boxes as [y_min, x_min, y_max, @@ -370,12 +370,12 @@ class Flux3AutoBlocks(SequentialPipelineBlocks): Indices of the text-encoder hidden layers whose states are stacked to form the text embeddings num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image_cond_latents (`list`, *optional*, required, depending on the workflow): + image_cond_latents (`list`, *optional*, depends on the workflow): Normalized packed latents of the reference images, one tensor of shape (B, C, height_i//16, width_i//16) per image - height (`int`, *optional*, defaults to None or 1024, depending on the workflow): + height (`int`, *optional*, depends on the workflow): The height in pixels of the generated image; defaults to the first reference image's own height - width (`int`, *optional*, defaults to None or 1024, depending on the workflow): + width (`int`, *optional*, depends on the workflow): The width in pixels of the generated image; defaults to the first reference image's own width reference_crop (`list | NoneType`, *optional*): Per reference, the box of the reference inside its padded canvas or None, from the preprocess step diff --git a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py index c68ff5435ee8..3f6ef7efc70b 100644 --- a/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py +++ b/src/diffusers/modular_pipelines/flux3/modular_blocks_flux3_distilled.py @@ -197,12 +197,12 @@ class Flux3DistilledAutoCoreDenoiseStep(AutoPipelineBlocks): The number of images to generate per prompt. prompt_embeds (`Tensor`): Pre-generated text embeddings. Can be generated from text_encoder step. - image_cond_latents (`list`, *optional*, required, depending on the workflow): + image_cond_latents (`list`, *optional*, depends on the workflow): Normalized packed latents of the reference images, one tensor of shape (B, C, height_i//16, width_i//16) per image - height (`int`, *optional*, defaults to None or 1024, depending on the workflow): + height (`int`, *optional*, depends on the workflow): The height in pixels of the generated image; defaults to the first reference image's own height - width (`int`, *optional*, defaults to None or 1024, depending on the workflow): + width (`int`, *optional*, depends on the workflow): The width in pixels of the generated image; defaults to the first reference image's own width reference_crop (`list | NoneType`, *optional*): Per reference, the box of the reference inside its padded canvas or None, from the preprocess step @@ -274,7 +274,7 @@ class Flux3DistilledAutoBlocks(SequentialPipelineBlocks): Inputs: prompt (`str`): The edit instruction to expand into the full caption (one per call). - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): The reference image(s) of the edit, at most 10; the same ones the pipeline receives. boxes (`list`, *optional*): Layout boxes, each {'id', 'desc', 'from', 'src_bbox', 'tgt_bbox'} with the boxes as [y_min, x_min, y_max, @@ -288,12 +288,12 @@ class Flux3DistilledAutoBlocks(SequentialPipelineBlocks): Indices of the text-encoder hidden layers whose states are stacked to form the text embeddings num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image_cond_latents (`list`, *optional*, required, depending on the workflow): + image_cond_latents (`list`, *optional*, depends on the workflow): Normalized packed latents of the reference images, one tensor of shape (B, C, height_i//16, width_i//16) per image - height (`int`, *optional*, defaults to None or 1024, depending on the workflow): + height (`int`, *optional*, depends on the workflow): The height in pixels of the generated image; defaults to the first reference image's own height - width (`int`, *optional*, defaults to None or 1024, depending on the workflow): + width (`int`, *optional*, depends on the workflow): The width in pixels of the generated image; defaults to the first reference image's own width reference_crop (`list | NoneType`, *optional*): Per reference, the box of the reference inside its padded canvas or None, from the preprocess step diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py index cd1e571bdd79..6839145fdc46 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios.py @@ -52,7 +52,7 @@ class HeliosAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) Inputs: - video (`None`, *optional*, required, depending on the workflow): + video (`None`, *optional*, depends on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -62,7 +62,7 @@ class HeliosAutoVaeEncoderStep(AutoPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. Outputs: @@ -364,7 +364,7 @@ class HeliosAutoCoreDenoiseStep(ConditionalPipelineBlocks): text embeddings used to guide the image generation. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): negative text embeddings used to guide the image generation. Can be generated from text_encoder step. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. @@ -468,7 +468,7 @@ class HeliosAutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 512): Maximum sequence length for prompt encoding. - video (`None`, *optional*, required, depending on the workflow): + video (`None`, *optional*, depends on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -478,11 +478,11 @@ class HeliosAutoBlocks(SequentialPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. num_videos_per_prompt (`int`, *optional*, defaults to 1): Number of videos to generate per prompt. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py index 03abbcb7876d..aa8a6ba3308c 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid.py @@ -51,7 +51,7 @@ class HeliosPyramidAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) Inputs: - video (`None`, *optional*, required, depending on the workflow): + video (`None`, *optional*, depends on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -61,7 +61,7 @@ class HeliosPyramidAutoVaeEncoderStep(AutoPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. Outputs: @@ -350,7 +350,7 @@ class HeliosPyramidAutoCoreDenoiseStep(ConditionalPipelineBlocks): text embeddings used to guide the image generation. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): negative text embeddings used to guide the image generation. Can be generated from text_encoder step. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. @@ -450,7 +450,7 @@ class HeliosPyramidAutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 512): Maximum sequence length for prompt encoding. - video (`None`, *optional*, required, depending on the workflow): + video (`None`, *optional*, depends on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -460,11 +460,11 @@ class HeliosPyramidAutoBlocks(SequentialPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. num_videos_per_prompt (`int`, *optional*, defaults to 1): Number of videos to generate per prompt. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. diff --git a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py index 6790965f4cc0..5f80fbbff3b5 100644 --- a/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py +++ b/src/diffusers/modular_pipelines/helios/modular_blocks_helios_pyramid_distilled.py @@ -51,7 +51,7 @@ class HeliosPyramidDistilledAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) Inputs: - video (`None`, *optional*, required, depending on the workflow): + video (`None`, *optional*, depends on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -61,7 +61,7 @@ class HeliosPyramidDistilledAutoVaeEncoderStep(AutoPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. Outputs: @@ -352,7 +352,7 @@ class HeliosPyramidDistilledAutoCoreDenoiseStep(ConditionalPipelineBlocks): text embeddings used to guide the image generation. Can be generated from text_encoder step. negative_prompt_embeds (`Tensor`, *optional*): negative text embeddings used to guide the image generation. Can be generated from text_encoder step. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. @@ -458,7 +458,7 @@ class HeliosPyramidDistilledAutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 512): Maximum sequence length for prompt encoding. - video (`None`, *optional*, required, depending on the workflow): + video (`None`, *optional*, depends on the workflow): Input video for video-to-video generation height (`int`, *optional*, defaults to 384): The height in pixels of the generated image. @@ -468,11 +468,11 @@ class HeliosPyramidDistilledAutoBlocks(SequentialPipelineBlocks): Number of latent frames per temporal chunk. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. num_videos_per_prompt (`int`, *optional*, defaults to 1): Number of videos to generate per prompt. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. video_latents (`Tensor`, *optional*): Encoded video latents for V2V generation. diff --git a/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py b/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py index ac40274b0dc5..0fb2f085a544 100644 --- a/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py +++ b/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py @@ -252,7 +252,7 @@ class HunyuanVideo15AutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLHunyuanVideo15`) video_processor (`HunyuanVideo15ImageProcessor`) Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. @@ -295,7 +295,7 @@ class HunyuanVideo15AutoImageEncoderStep(AutoPipelineBlocks): image_encoder (`SiglipVisionModel`) feature_extractor (`SiglipImageProcessor`) Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. Outputs: @@ -351,7 +351,7 @@ class HunyuanVideo15AutoCoreDenoiseStep(AutoPipelineBlocks): The number of images to generate per prompt. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): Pre-encoded image latents from the VAE encoder step, used as conditioning for I2V. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. @@ -412,7 +412,7 @@ class HunyuanVideo15AutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. @@ -431,7 +431,7 @@ class HunyuanVideo15AutoBlocks(SequentialPipelineBlocks): Pre-generated noisy latents for image generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): Pre-encoded image latents from the VAE encoder step, used as conditioning for I2V. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. @@ -485,7 +485,7 @@ class HunyuanVideo15Image2VideoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. diff --git a/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py b/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py index 42d8cce1f808..f1f491663c5a 100644 --- a/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py +++ b/src/diffusers/modular_pipelines/ltx/modular_blocks_ltx.py @@ -242,7 +242,7 @@ class LTXAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLLTXVideo`) video_processor (`VideoProcessor`) Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -311,7 +311,7 @@ class LTXAutoCoreDenoiseStep(AutoPipelineBlocks): Pre-generated noisy latents for image generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): TODO: Add description. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. @@ -356,7 +356,7 @@ class LTXAutoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 128): Maximum sequence length for prompt encoding. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -378,7 +378,7 @@ class LTXAutoBlocks(SequentialPipelineBlocks): TODO: Add description. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): TODO: Add description. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. @@ -433,7 +433,7 @@ class LTXImage2VideoBlocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 128): Maximum sequence length for prompt encoding. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py index b2b2ab0287c1..6da52ba54d88 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py @@ -68,7 +68,7 @@ class LTX2AutoPromptEnhancerStep(ConditionalPipelineBlocks): prompt_enhancer (`PreTrainedModel`) processor (`ProcessorMixin`) Inputs: - prompt (`str`, *optional*, required, depending on the workflow): + prompt (`str`, *optional*, depends on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -87,7 +87,7 @@ class LTX2AutoPromptEnhancerStep(ConditionalPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. Outputs: @@ -198,11 +198,11 @@ class LTX2AutoDurationStep(ConditionalPipelineBlocks): Upper bound on the auto-predicted duration. Must be strictly greater than `min_seconds`. frame_rate (`float`, *optional*, defaults to 24.0): Frames per second of the generated video. - connector_prompt_embeds (`Tensor`, *optional*, required, depending on the workflow): + connector_prompt_embeds (`Tensor`, *optional*, depends on the workflow): Video-branch text conditioning from the connector (positive prompt). - connector_audio_prompt_embeds (`Tensor`, *optional*, required, depending on the workflow): + connector_audio_prompt_embeds (`Tensor`, *optional*, depends on the workflow): Audio-branch text conditioning from the connector (positive prompt). - batch_size (`int`, *optional*, required, depending on the workflow): + batch_size (`int`, *optional*, depends on the workflow): The number of prompts being denoised, used to expand conditioning per prompt. Outputs: @@ -238,7 +238,7 @@ class LTX2AutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLLTX2Video`) text_encoder (`PreTrainedModel`) video_processor (`VideoProcessor`) Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*, defaults to 512): The height in pixels of the generated image. @@ -665,7 +665,7 @@ class LTX2AutoReferenceEncoderStep(ConditionalPipelineBlocks): vae (`AutoencoderKLLTX2Video`) transformer (`LTX2VideoTransformer3DModel`) video_processor (`VideoProcessor`) Inputs: - reference_conditions (`list`, *optional*, required, depending on the workflow): + reference_conditions (`list`, *optional*, depends on the workflow): `LTX2ReferenceCondition` (or list of them) whose videos are encoded into extra latent tokens the IC-LoRA adapter attends to. reference_downscale_factor (`int`, *optional*, defaults to 1): @@ -737,15 +737,15 @@ class LTX2AutoBuildVideoSelfAttentionMaskStep(ConditionalPipelineBlocks): - Skipped otherwise, leaving attention unmasked. Inputs: - latents (`Tensor`, *optional*, required, depending on the workflow): + latents (`Tensor`, *optional*, depends on the workflow): Pre-generated noisy latents for image generation. - base_token_count (`int`, *optional*, required, depending on the workflow): + base_token_count (`int`, *optional*, depends on the workflow): Number of generated-video tokens, i.e. the sequence length before appended tokens. - num_ref_tokens (`int`, *optional*, required, depending on the workflow): + num_ref_tokens (`int`, *optional*, depends on the workflow): Number of reference tokens, which sit at the very end of the sequence. - reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): + reference_cross_mask (`Tensor`, *optional*, depends on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. - reference_token_counts (`list`, *optional*, required, depending on the workflow): + reference_token_counts (`list`, *optional*, depends on the workflow): Per-reference token counts, used to split `reference_cross_mask` into attention groups. Outputs: @@ -860,7 +860,7 @@ class LTX2InContextCoreDenoiseStep(SequentialPipelineBlocks): The number of prompts being denoised, used to expand conditioning per prompt. generator (`Generator`, *optional*): Torch generator for deterministic generation. - reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): + reference_cross_mask (`Tensor`, *optional*, depends on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. @@ -953,13 +953,13 @@ class LTX2AutoCoreDenoiseStep(ConditionalPipelineBlocks): Audio-branch text conditioning (uncond). negative_connector_attention_mask (`Tensor`): Binary text attention mask (uncond). - condition_latents (`list`, *optional*, required, depending on the workflow): + condition_latents (`list`, *optional*, depends on the workflow): Per-condition normalized VAE latents of shape [1, C, F, H, W]. - condition_strengths (`list`, *optional*, required, depending on the workflow): + condition_strengths (`list`, *optional*, depends on the workflow): Per-condition conditioning strengths. - condition_indices (`list`, *optional*, required, depending on the workflow): + condition_indices (`list`, *optional*, depends on the workflow): Per-condition latent frame index at which the condition is applied. - condition_pixel_frames (`list`, *optional*, required, depending on the workflow): + condition_pixel_frames (`list`, *optional*, depends on the workflow): Per-condition trimmed pixel frame count, used to clamp single-frame keyframe coords. reference_conditions (`list`, *optional*): `LTX2ReferenceCondition` (or list of them); only their `strength` is read here. Omit for IC-LoRAs that @@ -995,7 +995,7 @@ class LTX2AutoCoreDenoiseStep(ConditionalPipelineBlocks): The number of prompts being denoised, used to expand conditioning per prompt. generator (`Generator`, *optional*): Torch generator for deterministic generation. - reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): + reference_cross_mask (`Tensor`, *optional*, depends on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. @@ -1011,7 +1011,7 @@ class LTX2AutoCoreDenoiseStep(ConditionalPipelineBlocks): Whether to condition the transformer on a separate per-token cross timestep (LTX-2.3+). attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): VAE-encoded reference-image latents used for image-to-video conditioning. Outputs: @@ -1197,7 +1197,7 @@ class LTX2AutoDecoderStep(AutoPipelineBlocks): Inputs: latents (`Tensor`): Pre-generated noisy latents for image generation. - base_token_count (`int`, *optional*, required, depending on the workflow): + base_token_count (`int`, *optional*, depends on the workflow): Number of generated-video tokens, i.e. the sequence length before appended tokens. output_type (`str`, *optional*, defaults to pil): Output format: 'pil', 'np', 'pt'. @@ -1268,7 +1268,7 @@ class LTX2Blocks(SequentialPipelineBlocks): (`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, required, depending on the workflow): + prompt (`str`, *optional*, depends on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1287,7 +1287,7 @@ class LTX2Blocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -1380,7 +1380,7 @@ class LTX2ImageToVideoBlocks(SequentialPipelineBlocks): audio_guider (`LTX2Guidance`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, required, depending on the workflow): + prompt (`str`, *optional*, depends on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1399,7 +1399,7 @@ class LTX2ImageToVideoBlocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -1501,7 +1501,7 @@ class LTX2ConditionBlocks(SequentialPipelineBlocks): (`LTX2Guidance`) video_processor (`VideoProcessor`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, required, depending on the workflow): + prompt (`str`, *optional*, depends on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1520,7 +1520,7 @@ class LTX2ConditionBlocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -1621,7 +1621,7 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): (`LTX2Guidance`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, required, depending on the workflow): + prompt (`str`, *optional*, depends on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1640,7 +1640,7 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -1653,7 +1653,7 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): num_frames (`int`, *optional*): The number of frames in the generated video. Omit to auto-predict via the `duration_head` (see `LTX2AutoDurationStep`). - reference_conditions (`list`, *optional*, required, depending on the workflow): + reference_conditions (`list`, *optional*, depends on the workflow): `LTX2ReferenceCondition` (or list of them) whose videos are encoded into extra latent tokens the IC-LoRA adapter attends to. reference_downscale_factor (`int`, *optional*, defaults to 1): @@ -1689,7 +1689,7 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): `sigmas` are supplied, else 1.0. sigmas (`list`, *optional*): Custom sigmas for the denoising process. - reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): + reference_cross_mask (`Tensor`, *optional*, depends on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. @@ -1777,7 +1777,7 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): audio_guider (`LTX2Guidance`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, required, depending on the workflow): + prompt (`str`, *optional*, depends on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1796,7 +1796,7 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -1820,7 +1820,7 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): num_frames (`int`, *optional*): The number of frames in the generated video. Omit to auto-predict via the `duration_head` (see `LTX2AutoDurationStep`). - reference_conditions (`list`, *optional*, required, depending on the workflow): + reference_conditions (`list`, *optional*, depends on the workflow): `LTX2ReferenceCondition` (or list of them) whose videos are encoded into extra latent tokens the IC-LoRA adapter attends to. reference_downscale_factor (`int`, *optional*, defaults to 1): @@ -1840,13 +1840,13 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): slow motion. `frame_rate` stays the playback rate. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - condition_latents (`list`, *optional*, required, depending on the workflow): + condition_latents (`list`, *optional*, depends on the workflow): Per-condition normalized VAE latents of shape [1, C, F, H, W]. - condition_strengths (`list`, *optional*, required, depending on the workflow): + condition_strengths (`list`, *optional*, depends on the workflow): Per-condition conditioning strengths. - condition_indices (`list`, *optional*, required, depending on the workflow): + condition_indices (`list`, *optional*, depends on the workflow): Per-condition latent frame index at which the condition is applied. - condition_pixel_frames (`list`, *optional*, required, depending on the workflow): + condition_pixel_frames (`list`, *optional*, depends on the workflow): Per-condition trimmed pixel frame count, used to clamp single-frame keyframe coords. reference_latents (`Tensor`, *optional*): Packed reference tokens of shape [1, total_reference_tokens, C], or `None` when no reference conditions @@ -1862,7 +1862,7 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): `sigmas` are supplied, else 1.0. sigmas (`list`, *optional*): Custom sigmas for the denoising process. - reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): + reference_cross_mask (`Tensor`, *optional*, depends on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. @@ -1876,9 +1876,9 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): Whether to condition the transformer on a separate per-token cross timestep (LTX-2.3+). attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): VAE-encoded reference-image latents used for image-to-video conditioning. - base_token_count (`int`, *optional*, required, depending on the workflow): + base_token_count (`int`, *optional*, depends on the workflow): Number of generated-video tokens, i.e. the sequence length before appended tokens. output_type (`str`, *optional*, defaults to pil): Output format: 'pil', 'np', 'pt'. diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py index 1c3a3d26f33b..2a3c40267812 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py @@ -162,7 +162,7 @@ class LTX25AutoDecoderStep(AutoPipelineBlocks): Inputs: latents (`Tensor`): Pre-generated noisy latents for image generation. - base_token_count (`int`, *optional*, required, depending on the workflow): + base_token_count (`int`, *optional*, depends on the workflow): Number of generated-video tokens, i.e. the sequence length before appended tokens. output_type (`str`, *optional*, defaults to pil): Output format: 'pil', 'np', 'pt'. @@ -233,7 +233,7 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): audio_guider (`LTX2Guidance`) diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, required, depending on the workflow): + prompt (`str`, *optional*, depends on the workflow): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -252,7 +252,7 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding). generator (`Generator`, *optional*): Torch generator for deterministic generation. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -276,7 +276,7 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): num_frames (`int`, *optional*): The number of frames in the generated video. Omit to auto-predict via the `duration_head` (see `LTX2AutoDurationStep`). - reference_conditions (`list`, *optional*, required, depending on the workflow): + reference_conditions (`list`, *optional*, depends on the workflow): `LTX2ReferenceCondition` (or list of them) whose videos are encoded into extra latent tokens the IC-LoRA adapter attends to. reference_downscale_factor (`int`, *optional*, defaults to 1): @@ -296,13 +296,13 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): slow motion. `frame_rate` stays the playback rate. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - condition_latents (`list`, *optional*, required, depending on the workflow): + condition_latents (`list`, *optional*, depends on the workflow): Per-condition normalized VAE latents of shape [1, C, F, H, W]. - condition_strengths (`list`, *optional*, required, depending on the workflow): + condition_strengths (`list`, *optional*, depends on the workflow): Per-condition conditioning strengths. - condition_indices (`list`, *optional*, required, depending on the workflow): + condition_indices (`list`, *optional*, depends on the workflow): Per-condition latent frame index at which the condition is applied. - condition_pixel_frames (`list`, *optional*, required, depending on the workflow): + condition_pixel_frames (`list`, *optional*, depends on the workflow): Per-condition trimmed pixel frame count, used to clamp single-frame keyframe coords. reference_latents (`Tensor`, *optional*): Packed reference tokens of shape [1, total_reference_tokens, C], or `None` when no reference conditions @@ -318,7 +318,7 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): `sigmas` are supplied, else 1.0. sigmas (`list`, *optional*): Custom sigmas for the denoising process. - reference_cross_mask (`Tensor`, *optional*, required, depending on the workflow): + reference_cross_mask (`Tensor`, *optional*, depends on the workflow): Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens]. num_inference_steps (`int`, *optional*, defaults to 30): The number of denoising steps. @@ -332,9 +332,9 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): Whether to condition the transformer on a separate per-token cross timestep (LTX-2.3+). attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): VAE-encoded reference-image latents used for image-to-video conditioning. - base_token_count (`int`, *optional*, required, depending on the workflow): + base_token_count (`int`, *optional*, depends on the workflow): Number of generated-video tokens, i.e. the sequence length before appended tokens. output_type (`str`, *optional*, defaults to pil): Output format: 'pil', 'np', 'pt'. diff --git a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py index b8a2d3a572b1..db5bf991687a 100644 --- a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py +++ b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py @@ -56,7 +56,7 @@ class MiniMaxH3AutoBeforeEncodeStep(ConditionalPipelineBlocks): 2048) Inputs: - references (`list`, *optional*, required, depending on the workflow): + references (`list`, *optional*, depends on the workflow): The references to condition on, **in the order the model should read them**: the order labels them in the prompt presentation and lays them out on the shared rotary clock, so a different order is a different request. One dataclass per modality, all holding in-memory media — a [`MiniMaxH3ImageReference`] (at most @@ -68,7 +68,7 @@ class MiniMaxH3AutoBeforeEncodeStep(ConditionalPipelineBlocks): Height of the generated video in pixels, a multiple of 32. width (`int`, *optional*): Width of the generated video in pixels, a multiple of 32. - num_frames (`int`, *optional*, required, depending on the workflow): + num_frames (`int`, *optional*, depends on the workflow): Number of frames to generate, at the fixed 24 fps. Snapped up to the next `17 * n + 5` the video VAE can decode; the resulting duration must stay between 5 and 15 seconds. To generate a video as long as a reference soundtrack, pass `round(samples / sample_rate * 24)`. @@ -140,7 +140,7 @@ class MiniMaxH3AutoTextEncoderStep(ConditionalPipelineBlocks): Inputs: prompt (`str`): The prompt to guide generation, a single string. - normalized_references (`list`, *optional*, required, depending on the workflow): + normalized_references (`list`, *optional*, depends on the workflow): The references normalized by the setup step, in packed order. keyframes (`list`, *optional*): The keyframes put onto the target canvas, in packed order. @@ -191,9 +191,9 @@ class MiniMaxH3AutoVaeEncoderStep(ConditionalPipelineBlocks): vae (`AutoencoderKLMiniMaxH3`) audio_vae (`AutoencoderKLMiniMaxH3Audio`) Inputs: - normalized_references (`list`, *optional*, required, depending on the workflow): + normalized_references (`list`, *optional*, depends on the workflow): The references normalized by the setup step, in packed order. - keyframes (`list`, *optional*, required, depending on the workflow): + keyframes (`list`, *optional*, depends on the workflow): The keyframes put onto the target canvas, in packed order. Outputs: @@ -543,18 +543,18 @@ class MiniMaxH3AutoDenoiseStep(ConditionalPipelineBlocks): Inputs: text_token_tags (`Tensor`): The per-row modality tag of every row of `prompt_embeds`. - normalized_references (`list`, *optional*, required, depending on the workflow): + normalized_references (`list`, *optional*, depends on the workflow): The references normalized by the setup step, in packed order. - condition_latents (`list`, *optional*, required, depending on the workflow): + condition_latents (`list`, *optional*, depends on the workflow): The encoded video conditioning latents, one per image and video reference in packed order. Their shape is where every reference block's geometry comes from. - audio_condition_latents (`list`, *optional*, required, depending on the workflow): + audio_condition_latents (`list`, *optional*, depends on the workflow): The encoded audio conditioning rows, one per audio-bearing reference in packed order. - height (`int`, *optional*, required or defaults to None, depending on the workflow): + height (`int`, *optional*, depends on the workflow): Height of the generated video in pixels. - width (`int`, *optional*, required or defaults to None, depending on the workflow): + width (`int`, *optional*, depends on the workflow): Width of the generated video in pixels. - num_frames (`int`, *optional*, required or defaults to 124, depending on the workflow): + num_frames (`int`, *optional*, depends on the workflow): Resolved number of frames, of the form 17 * n + 5. generator (`Generator`, *optional*): The generator of the request. The conditioning noise is drawn from it first, one draw per condition, @@ -680,7 +680,7 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): 2048) Inputs: - references (`list`, *optional*, required, depending on the workflow): + references (`list`, *optional*, depends on the workflow): The references to condition on, **in the order the model should read them**: the order labels them in the prompt presentation and lays them out on the shared rotary clock, so a different order is a different request. One dataclass per modality, all holding in-memory media — a [`MiniMaxH3ImageReference`] (at most @@ -704,14 +704,14 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): it is the follower of the two and is cover-cropped onto the canvas. prompt (`str`): The prompt to guide generation, a single string. - normalized_references (`list`, *optional*, required, depending on the workflow): + normalized_references (`list`, *optional*, depends on the workflow): The references normalized by the setup step, in packed order. keyframes (`list`, *optional*): The keyframes put onto the target canvas, in packed order. - condition_latents (`list`, *optional*, required, depending on the workflow): + condition_latents (`list`, *optional*, depends on the workflow): The encoded video conditioning latents, one per image and video reference in packed order. Their shape is where every reference block's geometry comes from. - audio_condition_latents (`list`, *optional*, required, depending on the workflow): + audio_condition_latents (`list`, *optional*, depends on the workflow): The encoded audio conditioning rows, one per audio-bearing reference in packed order. generator (`Generator`, *optional*): The generator of the request. The conditioning noise is drawn from it first, one draw per condition, diff --git a/src/diffusers/modular_pipelines/p_h3_image/prompt_upsampler.py b/src/diffusers/modular_pipelines/p_h3_image/prompt_upsampler.py index 60302b9d4439..5aefee601b32 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/prompt_upsampler.py +++ b/src/diffusers/modular_pipelines/p_h3_image/prompt_upsampler.py @@ -264,9 +264,9 @@ class PH3ImageAutoPromptUpsampleStep(AutoPipelineBlocks): Inputs: prompt (`str`): The prompt, or the edit instruction when `image` is passed. A single string. - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, depends on the workflow): The images of the edit, at most 8, the image to edit first; the same ones the pipeline receives. - max_new_tokens (`int`, *optional*, defaults to 1536 or 1024, depending on the workflow): + max_new_tokens (`int`, *optional*, depends on the workflow): The most tokens the upsampler may generate, its reasoning included. height (`int`, *optional*, defaults to 2912): The height in pixels of the generated image. diff --git a/src/diffusers/modular_pipelines/pika_h3_flex/modular_blocks_pika_h3_flex.py b/src/diffusers/modular_pipelines/pika_h3_flex/modular_blocks_pika_h3_flex.py index 06dd9265a3c6..fb126c4b6343 100644 --- a/src/diffusers/modular_pipelines/pika_h3_flex/modular_blocks_pika_h3_flex.py +++ b/src/diffusers/modular_pipelines/pika_h3_flex/modular_blocks_pika_h3_flex.py @@ -55,7 +55,7 @@ class PikaH3FlexAutoBeforeEncodeStep(ConditionalPipelineBlocks): canvas_short_edge (default: 768) canvas_max_pixels (default: 1032192) Inputs: - references (`list`, *optional*): + references (`list`, *optional*, depends on the workflow): The references to condition on, **in the order the model should read them**: the order labels them in the prompt presentation and lays them out on the shared rotary clock, so a different order is a different request. One dataclass per modality, all holding in-memory media — a [`PikaH3FlexImageReference`] (at @@ -69,7 +69,7 @@ class PikaH3FlexAutoBeforeEncodeStep(ConditionalPipelineBlocks): Height of the generated video in pixels, a multiple of 32. width (`int`, *optional*): Width of the generated video in pixels, a multiple of 32. - num_frames (`int`, *optional*): + num_frames (`int`, *optional*, depends on the workflow): Number of frames to generate, at the fixed 24 fps: 120 to 360, i.e. 5 to 15 seconds, snapped up to the next `17 * n + 5` the video VAE can decode. To generate a video as long as a reference soundtrack, pass `round(samples / sample_rate * 24)`. @@ -143,7 +143,7 @@ class PikaH3FlexAutoTextEncoderStep(ConditionalPipelineBlocks): Inputs: prompt (`str`): The prompt to guide generation, a single string. - normalized_references (`list`, *optional*): + normalized_references (`list`, *optional*, depends on the workflow): The references normalized by the setup step, in packed order. keyframes (`list`, *optional*): The keyframes put onto the target canvas, in packed order. @@ -195,9 +195,9 @@ class PikaH3FlexAutoVaeEncoderStep(ConditionalPipelineBlocks): vae (`AutoencoderKLMiniMaxH3`) audio_vae (`AutoencoderKLMiniMaxH3Audio`) Inputs: - normalized_references (`list`, *optional*): + normalized_references (`list`, *optional*, depends on the workflow): The references normalized by the setup step, in packed order. - keyframes (`list`, *optional*): + keyframes (`list`, *optional*, depends on the workflow): The keyframes put onto the target canvas, in packed order. Outputs: @@ -550,26 +550,26 @@ class PikaH3FlexAutoDenoiseStep(ConditionalPipelineBlocks): Inputs: text_token_tags (`Tensor`): The per-row modality tag of every row of `prompt_embeds`. - normalized_references (`list`, *optional*): + normalized_references (`list`, *optional*, depends on the workflow): The references normalized by the setup step, in packed order. - condition_latents (`list`, *optional*): + condition_latents (`list`, *optional*, depends on the workflow): The encoded video conditioning latents, one per image and video reference in packed order. Their shape is where every reference block's geometry comes from. - audio_condition_latents (`list`, *optional*): + audio_condition_latents (`list`, *optional*, depends on the workflow): The encoded audio conditioning rows, one per audio-bearing reference in packed order. - height (`int`, *optional*): + height (`int`, *optional*, depends on the workflow): Height of the generated video in pixels. - width (`int`, *optional*): + width (`int`, *optional*, depends on the workflow): Width of the generated video in pixels. - num_frames (`int`, *optional*, defaults to None or 124, depending on the workflow): + num_frames (`int`, *optional*, depends on the workflow): Resolved number of frames, of the form 17 * n + 5. generator (`Generator`, *optional*): The generator of the request. The conditioning noise is drawn from it first, one draw per condition, before the noise of the generated rows. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated video noise of shape `(1, 24, num_latent_frames, latent_height, latent_width)`, used instead of the draw. - audio_latents (`Tensor`): + audio_latents (`Tensor`, *optional*): Pre-generated audio noise of shape `(2, 32, num_audio_latents)`. prompt_embeds (`Tensor`): text embeddings used to guide the image generation. Can be generated from text_encoder step. @@ -691,7 +691,7 @@ class PikaH3FlexBlocks(SequentialPipelineBlocks): 750, 500, 250, 100, 40, 0]) Inputs: - references (`list`, *optional*): + references (`list`, *optional*, depends on the workflow): The references to condition on, **in the order the model should read them**: the order labels them in the prompt presentation and lays them out on the shared rotary clock, so a different order is a different request. One dataclass per modality, all holding in-memory media — a [`PikaH3FlexImageReference`] (at @@ -705,7 +705,7 @@ class PikaH3FlexBlocks(SequentialPipelineBlocks): Height of the generated video in pixels, a multiple of 32. width (`int`, *optional*): Width of the generated video in pixels, a multiple of 32. - num_frames (`int`, *optional*): + num_frames (`int`, *optional*, depends on the workflow): Number of frames to generate, at the fixed 24 fps: 120 to 360, i.e. 5 to 15 seconds, snapped up to the next `17 * n + 5` the video VAE can decode. To generate a video as long as a reference soundtrack, pass `round(samples / sample_rate * 24)`. @@ -717,22 +717,22 @@ class PikaH3FlexBlocks(SequentialPipelineBlocks): the canvas' aspect ratio and resized onto it, like `image`. prompt (`str`): The prompt to guide generation, a single string. - normalized_references (`list`, *optional*): + normalized_references (`list`, *optional*, depends on the workflow): The references normalized by the setup step, in packed order. keyframes (`list`, *optional*): The keyframes put onto the target canvas, in packed order. - condition_latents (`list`, *optional*): + condition_latents (`list`, *optional*, depends on the workflow): The encoded video conditioning latents, one per image and video reference in packed order. Their shape is where every reference block's geometry comes from. - audio_condition_latents (`list`, *optional*): + audio_condition_latents (`list`, *optional*, depends on the workflow): The encoded audio conditioning rows, one per audio-bearing reference in packed order. generator (`Generator`, *optional*): The generator of the request. The conditioning noise is drawn from it first, one draw per condition, before the noise of the generated rows. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated video noise of shape `(1, 24, num_latent_frames, latent_height, latent_width)`, used instead of the draw. - audio_latents (`Tensor`): + audio_latents (`Tensor`, *optional*): Pre-generated audio noise of shape `(2, 32, num_audio_latents)`. **denoiser_input_fields (`None`, *optional*): The structural description of the packed sequence the transformer reads by name: `token_tags`, diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py index a6a19d36b94f..1e3d63bb1bbd 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py @@ -70,7 +70,7 @@ class QwenImageAutoTextEncoderStep(AutoPipelineBlocks): The tokenizer to use guider (`ClassifierFreeGuidance`) Inputs: - prompt (`str`, *optional*, required, depending on the workflow): + prompt (`str`, *optional*, depends on the workflow): The prompt or prompts to guide image generation. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. @@ -220,7 +220,7 @@ class QwenImageOptionalControlNetVaeEncoderStep(AutoPipelineBlocks): (`VaeImageProcessor`) Inputs: - control_image (`Image`, *optional*, required, depending on the workflow): + control_image (`Image`, *optional*, depends on the workflow): Control image for ControlNet conditioning. height (`int`, *optional*): The height in pixels of the generated image. @@ -1130,15 +1130,15 @@ class QwenImageAutoBlocks(SequentialPipelineBlocks): (`FlowMatchEulerDiscreteScheduler`) transformer (`QwenImageTransformer2DModel`) Inputs: - prompt (`str`, *optional*, required, depending on the workflow): + prompt (`str`, *optional*, depends on the workflow): The prompt or prompts to guide image generation. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. max_sequence_length (`int`, *optional*, defaults to 1024): Maximum sequence length for prompt encoding. - mask_image (`Image`, *optional*, required, depending on the workflow): + mask_image (`Image`, *optional*, depends on the workflow): Mask image for inpainting. - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. @@ -1148,7 +1148,7 @@ class QwenImageAutoBlocks(SequentialPipelineBlocks): Padding for mask cropping in inpainting. generator (`Generator`, *optional*): Torch generator for deterministic generation. - control_image (`Image`, *optional*, required, depending on the workflow): + control_image (`Image`, *optional*, depends on the workflow): Control image for ControlNet conditioning. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. @@ -1170,13 +1170,13 @@ class QwenImageAutoBlocks(SequentialPipelineBlocks): Additional kwargs for attention processors. **denoiser_input_fields (`None`, *optional*): conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): image latents used to guide the image generation. Can be generated from vae_encoder step. processed_mask_image (`Tensor`, *optional*): The processed mask image strength (`float`, *optional*, defaults to 0.9): Strength for img2img/inpainting. - control_image_latents (`Tensor`, *optional*, required, depending on the workflow): + control_image_latents (`Tensor`, *optional*, depends on the workflow): The control image latents to use for the denoising process. Can be generated in controlnet vae encoder step. control_guidance_start (`float`, *optional*, defaults to 0.0): diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py index 689bf8516b0d..9ea44e5a847a 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage_edit.py @@ -736,7 +736,7 @@ class QwenImageEditAutoBlocks(SequentialPipelineBlocks): The prompt or prompts to guide image generation. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. - mask_image (`Image`, *optional*, required, depending on the workflow): + mask_image (`Image`, *optional*, depends on the workflow): Mask image for inpainting. padding_mask_crop (`int`, *optional*): Padding for mask cropping in inpainting. diff --git a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py index 87a63adb513e..7df70b28ea18 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py +++ b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21.py @@ -108,7 +108,7 @@ class QwenImage21AutoTextEncoderStep(AutoPipelineBlocks): (`Qwen3VLProcessor`) Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -208,7 +208,7 @@ class QwenImage21AutoVaeEncoderStep(AutoPipelineBlocks): image_processor (`VaeImageProcessor`) vae (`AutoencoderKLQwenImage21`) Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -438,7 +438,7 @@ class QwenImage21AutoCoreDenoiseStep(AutoPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`list`, *optional*, required, depending on the workflow): + image_latents (`list`, *optional*, depends on the workflow): Normalized latents of each condition image. Can be generated from vae_encoder step. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. @@ -510,7 +510,7 @@ class QwenImage21AutoBlocks(SequentialPipelineBlocks): (`ClassifierFreeGuidance`) transformer (`QwenImage21Transformer2DModel`) Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -526,7 +526,7 @@ class QwenImage21AutoBlocks(SequentialPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`list`, *optional*, required, depending on the workflow): + image_latents (`list`, *optional*, depends on the workflow): Normalized latents of each condition image. Can be generated from vae_encoder step. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. diff --git a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py index 276e471ae962..7fa3d50baaf3 100644 --- a/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py +++ b/src/diffusers/modular_pipelines/qwenimage21/modular_blocks_qwenimage21_turbo.py @@ -100,7 +100,7 @@ class QwenImage21TurboAutoTextEncoderStep(AutoPipelineBlocks): (`Qwen3VLProcessor`) Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -192,7 +192,7 @@ class QwenImage21TurboAutoVaeEncoderStep(AutoPipelineBlocks): image_processor (`VaeImageProcessor`) vae (`AutoencoderKLQwenImage21`) Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -407,7 +407,7 @@ class QwenImage21TurboAutoCoreDenoiseStep(AutoPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`list`, *optional*, required, depending on the workflow): + image_latents (`list`, *optional*, depends on the workflow): Normalized latents of each condition image. Can be generated from vae_encoder step. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. @@ -483,7 +483,7 @@ class QwenImage21TurboAutoBlocks(SequentialPipelineBlocks): sample_sigmas (default: None): Sampling grid the checkpoint ships with, used when `sigmas` is not passed. Inputs: - image (`Image | list`, *optional*, required, depending on the workflow): + image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. output_resolution (`int`, *optional*, defaults to 1024): Target side length used to derive the output size and to resize condition images. @@ -497,7 +497,7 @@ class QwenImage21TurboAutoBlocks(SequentialPipelineBlocks): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - image_latents (`list`, *optional*, required, depending on the workflow): + image_latents (`list`, *optional*, depends on the workflow): Normalized latents of each condition image. Can be generated from vae_encoder step. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. diff --git a/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py b/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py index f93abec2cb04..716443369307 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py @@ -241,7 +241,7 @@ class StableDiffusion3AutoCoreDenoiseStep(AutoPipelineBlocks): The height in pixels of the generated image. width (`None`, *optional*): The width in pixels of the generated image. - image_latents (`None`, *optional*, required, depending on the workflow): + image_latents (`None`, *optional*, depends on the workflow): Latent input image_latents to be processed. latents (`Tensor | NoneType`, *optional*): Pre-generated noisy latents to be used as inputs for image generation. @@ -330,7 +330,7 @@ class StableDiffusion3AutoBlocks(SequentialPipelineBlocks): One or a list of torch generator(s) to make generation deterministic. num_images_per_prompt (`None`, *optional*, defaults to 1): The number of images to generate per prompt. - image_latents (`None`, *optional*, required, depending on the workflow): + image_latents (`None`, *optional*, depends on the workflow): Latent input image_latents to be processed. latents (`Tensor | NoneType`, *optional*): Pre-generated noisy latents to be used as inputs for image generation. diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py index f0ef07948570..45f0acfbff97 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py @@ -332,15 +332,15 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): TODO: Add description. clip_skip (`None`, *optional*): TODO: Add description. - ip_adapter_image (`Image | ndarray | Tensor | list | list | list`, *optional*, required, depending on the workflow): + ip_adapter_image (`Image | ndarray | Tensor | list | list | list`, *optional*, depends on the workflow): The image(s) to be used as ip adapter height (`None`, *optional*): TODO: Add description. width (`None`, *optional*): TODO: Add description. - image (`None`, *optional*, required, depending on the workflow): + image (`None`, *optional*, depends on the workflow): TODO: Add description. - mask_image (`None`, *optional*, required, depending on the workflow): + mask_image (`None`, *optional*, depends on the workflow): TODO: Add description. padding_mask_crop (`None`, *optional*): TODO: Add description. @@ -365,16 +365,16 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): TODO: Add description. denoising_end (`None`, *optional*): TODO: Add description. - strength (`float`, *optional*, defaults to 0.9999 or 0.3, depending on the workflow): + strength (`float`, *optional*, depends on the workflow): Strength for img2img/inpainting. denoising_start (`None`, *optional*): TODO: Add description. latents (`None`, *optional*): TODO: Add description. - image_latents (`Tensor`, *optional*, required, depending on the workflow): + image_latents (`Tensor`, *optional*, depends on the workflow): The latents representing the reference image for image-to-image/inpainting generation. Can be generated in vae_encode step. - mask (`Tensor`, *optional*, required, depending on the workflow): + mask (`Tensor`, *optional*, depends on the workflow): The mask for the inpainting generation. Can be generated in vae_encode step. masked_image_latents (`Tensor`, *optional*): The masked image latents for the inpainting generation (only for inpainting-specific unet). Can be @@ -395,9 +395,9 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): TODO: Add description. negative_aesthetic_score (`None`, *optional*, defaults to 2.0): TODO: Add description. - control_image (`None`, *optional*, required, depending on the workflow): + control_image (`None`, *optional*, depends on the workflow): TODO: Add description. - control_mode (`None`, *optional*, required, depending on the workflow): + control_mode (`None`, *optional*, depends on the workflow): TODO: Add description. control_guidance_start (`None`, *optional*, defaults to 0.0): TODO: Add description. @@ -410,12 +410,12 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): crops_coords (`tuple | NoneType`, *optional*): The crop coordinates to use for preprocess/postprocess the image and mask, for inpainting task only. Can be generated in vae_encode step. - controlnet_cond (`Tensor`, *optional*, required, depending on the workflow): + controlnet_cond (`Tensor`, *optional*, depends on the workflow): The control image to use for the denoising process. Can be generated in prepare_controlnet_inputs step. conditioning_scale (`float`, *optional*): The controlnet conditioning scale value to use for the denoising process. Can be generated in prepare_controlnet_inputs step. - controlnet_keep (`list`, *optional*, required, depending on the workflow): + controlnet_keep (`list`, *optional*, depends on the workflow): The controlnet keep values to use for the denoising process. Can be generated in prepare_controlnet_inputs step. **denoiser_input_fields (`None`, *optional*): diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py index 06f6abd90e68..ac834eb37d48 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py @@ -137,13 +137,13 @@ class WanAutoImageEncoderStep(AutoPipelineBlocks): image_processor (`CLIPImageProcessor`) image_encoder (`CLIPVisionModel`) Inputs: - image (`Image`, *optional*, required, depending on the workflow): + image (`Image`, *optional*, depends on the workflow): TODO: Add description. height (`int`, *optional*, defaults to 480): TODO: Add description. width (`int`, *optional*, defaults to 832): TODO: Add description. - last_image (`Image`, *optional*, required, depending on the workflow): + last_image (`Image`, *optional*, depends on the workflow): The last frameimage Outputs: @@ -290,13 +290,13 @@ class WanAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) Inputs: - image (`Image`, *optional*, required, depending on the workflow): + image (`Image`, *optional*, depends on the workflow): TODO: Add description. height (`int`, *optional*, defaults to 480): TODO: Add description. width (`int`, *optional*, defaults to 832): TODO: Add description. - last_image (`Image`, *optional*, required, depending on the workflow): + last_image (`Image`, *optional*, depends on the workflow): The last frameimage num_frames (`int`, *optional*, defaults to 81): TODO: Add description. @@ -447,13 +447,13 @@ class WanImage2VideoAutoBlocks(SequentialPipelineBlocks): TODO: Add description. max_sequence_length (`None`, *optional*, defaults to 512): TODO: Add description. - image (`Image`, *optional*, required, depending on the workflow): + image (`Image`, *optional*, depends on the workflow): TODO: Add description. height (`int`, *optional*, defaults to 480): The target height of the video. width (`int`, *optional*, defaults to 832): The target width of the video. - last_image (`Image`, *optional*, required, depending on the workflow): + last_image (`Image`, *optional*, depends on the workflow): The last frameimage num_frames (`int`, *optional*, defaults to 81): The number of frames in the generated video. diff --git a/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py b/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py index 66565b260262..58253c903b62 100644 --- a/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py +++ b/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py @@ -188,7 +188,7 @@ class ZImageAutoDenoiseStep(AutoPipelineBlocks): TODO: Add description. width (`None`, *optional*): TODO: Add description. - image_latents (`None`, *optional*, required, depending on the workflow): + image_latents (`None`, *optional*, depends on the workflow): TODO: Add description. latents (`Tensor | NoneType`, *optional*): TODO: Add description. @@ -236,7 +236,7 @@ class ZImageAutoVaeImageEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKL`) image_processor (`VaeImageProcessor`) Inputs: - image (`Image`, *optional*, required, depending on the workflow): + image (`Image`, *optional*, depends on the workflow): TODO: Add description. height (`None`, *optional*): TODO: Add description. @@ -283,7 +283,7 @@ class ZImageAutoBlocks(SequentialPipelineBlocks): TODO: Add description. max_sequence_length (`None`, *optional*, defaults to 512): TODO: Add description. - image (`Image`, *optional*, required, depending on the workflow): + image (`Image`, *optional*, depends on the workflow): TODO: Add description. height (`None`, *optional*): TODO: Add description. @@ -293,7 +293,7 @@ class ZImageAutoBlocks(SequentialPipelineBlocks): TODO: Add description. num_images_per_prompt (`None`, *optional*, defaults to 1): TODO: Add description. - image_latents (`None`, *optional*, required, depending on the workflow): + image_latents (`None`, *optional*, depends on the workflow): TODO: Add description. latents (`Tensor | NoneType`, *optional*): TODO: Add description. From c2667549b08db3af03752f1b5f3a11cd457a1a98 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sun, 8 Nov 2026 10:46:53 +0100 Subject: [PATCH 07/33] [docs] modular: how a composite derives inputs and when to use `*_overrides` Co-Authored-By: Claude Fable 5.1 --- .ai/references/modular.md | 57 +++++++++---------- .../sequential_pipeline_blocks.md | 27 +++------ 2 files changed, 35 insertions(+), 49 deletions(-) diff --git a/.ai/references/modular.md b/.ai/references/modular.md index e298997915bd..55f97284b18c 100644 --- a/.ai/references/modular.md +++ b/.ai/references/modular.md @@ -269,7 +269,7 @@ OutputParam( ) ``` -If a template's predefined description doesn't fit (e.g. the `"latents"` output template means "Denoised latents", which is wrong for the noisy latents out of a prepare-latents step) — drop the template and declare the field directly with an accurate description. See gotcha #5. +If a template's predefined description doesn't fit (e.g. the `"latents"` output template means "Denoised latents", which is wrong for the noisy latents out of a prepare-latents step) — keep the template for the name and type hint and pass your own `description=`, or declare the field directly. See gotcha #5. **Declare defaults in the `InputParam`, not inside `__call__`.** @@ -297,32 +297,10 @@ if block_state.height is None: ``` The other two ways to declare an input: -(1) `required=True` when the block cannot work without the value, so the caller has to pass it; the block raises `ValueError("Required input 'x' is missing")` when it is not passed or `None`. -(2) Optional with no default when the block works with `None`. Most often that is the computed case above, and the block should then also declare the value as an output. Sometimes the block only uses the value to decide something else: e.g. in the text encoder blocks, `negative_prompt` defaults to `None` instead of `""`, so the encoder can tell "none given" from an empty string. It encodes an empty string in its place only when guidance needs one, and ignores a given negative prompt when guidance is disabled (some encoders log a warning). +(1) `required=True` when the block cannot work without the value, so the caller has to pass it; the block raises `ValueError("Required input 'x' is missing")` when it is not passed or `None`. +(2) Optional with no default when the block works with `None`. Most often that is the computed case above, and the block should then also declare the value as an output. -Inputs are declared on leaf blocks. When blocks are assembled, the composite derives its inputs from them, and the same input is sometimes declared by more than one sub-block. - -In a conditional block, the sub-blocks may declare different defaults for the same input, and each is respected: e.g. for `strength`, the img2img branch may defaults to 0.3 and the inpaint branch to 0.9999, the docstring of the conditional block says "defaults to 0.3 or 0.9999, depending on the workflow", and at runtime the branch that runs fills in its own. - -In a sequential block, the first sub-block that declares a default sets it for every sub-block after it. When a later sub-block declares a different default, the sequence keeps the first one and warns that the default is ambiguous. We recommend to state it explicitly for such case: declare the input in `input_overrides` on the composite, see below. - -**Redefine what the composite derives with `*_overrides`.** When you assemble blocks, the composite derives its inputs, outputs, components and configs from its sub-blocks. You cannot add or remove any of them there, they are part of the blocks' logic, but you can redefine how one is declared: e.g. you can change the default and description of an input, the description of an output, the repo and loading info of a component, the value of a config. Set `input_overrides`, `intermediate_output_overrides`, `component_overrides` or `config_overrides` as a class attribute on the composite (`ConditionalPipelineBlocks`, `SequentialPipelineBlocks`, `LoopSequentialPipelineBlocks`), or pass them to `from_blocks_dict`: - -```python -# as a class attribute -class MyBlocks(SequentialPipelineBlocks): - block_classes = [TextEncoderStep, DenoiseStep] - block_names = ["text_encoder", "denoise"] - input_overrides = {"num_inference_steps": InputParam.template("num_inference_steps", default=10)} - -# or when assembling from a dict -blocks = SequentialPipelineBlocks.from_blocks_dict( - {"text_encoder": TextEncoderStep(), "denoise": DenoiseStep()}, - input_overrides={"num_inference_steps": InputParam.template("num_inference_steps", default=10)}, -) -``` - -Each is a dict of name to declaration. The name must be one the composite already derives, and the declaration, an `InputParam` / `OutputParam` / `ComponentSpec` / `ConfigSpec`, replaces the derived one whole, so it states everything: default, `required`, description. Typical uses: settle a default two sub-blocks disagree on, set a preset's default as in the distilled example below, or reword a description from the composite's point of view. The per-branch defaults of a conditional block cannot be changed one branch at a time from the composite; for that, change the sub-block. An override on the conditional block replaces them with one default for every branch. `InputParam.template(...)` keeps a common input to one line. +**An assembled block derives its declarations from its sub-blocks, and can redefine them with `*_overrides`.** Inputs, outputs, components and configs are declared on leaf blocks. A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its own from its sub-blocks. It cannot add or remove any of them, they are part of the blocks' logic, but it can redefine how one is declared: the default or description of an input, the description of an output, the repo and loading info of a component, the value of a config. You set these as class attributes on the composite, or pass them to `from_blocks_dict`: `input_overrides`, `intermediate_output_overrides`, `component_overrides` and `config_overrides`, each a `{name: declaration}` dict. The name must be one the composite already derives, and the declaration takes the derived entry's place (gotcha #13). ```python class DistilledCoreDenoiseStep(SequentialPipelineBlocks): @@ -332,9 +310,15 @@ class DistilledCoreDenoiseStep(SequentialPipelineBlocks): input_overrides = {"num_inference_steps": InputParam.template("num_inference_steps", default=10)} ``` -A composite reads its sub-blocks' declarations and never writes to them: the `required` flag it reports is its own view ("the caller has to pass this"), the leaf's flag is the runtime check in `get_block_state`, and the two legitimately differ. Copy an `InputParam` before changing it. +`component_overrides` and `config_overrides` work the same way with a `ComponentSpec` / `ConfigSpec`, see the ComponentSpec section below. Typical uses: set a preset's default as above, settle a default two sub-blocks disagree on, or reword a description from the composite's point of view. + +How the composite derives a shared input, and when you need an override: -Overriding the `outputs` property by hand is still the way to narrow `outputs` to what downstream actually consumes, so the docstring shows the step's product instead of every internal intermediate (Wan-Animate-2's core denoise exposes only `segment_frames`). +- In a conditional block, the sub-blocks may declare different defaults for the same input, and each is respected: e.g. for `strength`, the img2img branch defaults to 0.3 and the inpaint branch to 0.9999, the docstring of the conditional block says "depends on the workflow", and at runtime the branch that runs fills in its own. An `input_overrides` entry on the conditional block will gives every branch the same default and cannot set one per branch. To change a single branch's default, override the input on that sub-block. +- In a sequential block, the first sub-block that declares a default sets it for every sub-block after it. When a later sub-block declares a different default, the sequence keeps the first one and warns that the default is ambiguous. State it explicitly: declare the input in `input_overrides` on the composite. +- The `required` flag the composite reports is its own view, whether the caller has to pass the value. A sub-block may require an input that an earlier sub-block defaults or may produce; the composite then lists it as optional. A conditional block lists an input as required only when every branch requires it and the block always runs. The sub-blocks' own declarations are untouched and still checked at runtime in `get_block_state`. + +**`outputs` is what the assembled block presents as its product.** By default it is every intermediate output. Override the property to list only what the step is for, so the docstring shows the product instead of every internal intermediate (Wan-Animate-2's core denoise lists only `segment_frames`). It changes the docstring and `output_names` only: everything stays in the pipeline state, and `pipe(..., output="latents")` still fetches any intermediate. ## ComponentSpec patterns @@ -352,6 +336,19 @@ ComponentSpec( ) ``` +A composite inherits its sub-blocks' `ComponentSpec`s and `ConfigSpec`s. To point one at another repo, loading method or config value for the assembled pipeline, redeclare it in `component_overrides` / `config_overrides` on the composite, the same way `input_overrides` redeclares an input (see InputParam / OutputParam above). + +```python +class DistilledBlocks(SequentialPipelineBlocks): + block_classes = [TextEncoderStep, DenoiseStep, DecodeStep] + block_names = ["text_encoder", "denoise", "decode"] + # the distilled checkpoint has its own transformer weights and skips guidance + component_overrides = { + "transformer": ComponentSpec("transformer", YourTransformerModel, repo="org/model-distilled", subfolder="transformer"), + } + config_overrides = {"is_distilled": ConfigSpec("is_distilled", default=True)} +``` + ## Gotchas 1. **Importing from standard pipelines.** The modular and standard pipeline systems are parallel — modular blocks must not import from `diffusers.pipelines.*`. For shared utility methods (e.g. `_pack_latents`, `retrieve_timesteps`), either redefine as standalone functions or use `# Copied from diffusers.pipelines....` headers. See `wan/before_denoise.py` and `helios/before_denoise.py` for examples. @@ -362,7 +359,7 @@ ComponentSpec( 4. **Instantiating components inline.** If a class like `VideoProcessor` is needed, register it as a `ComponentSpec` and access via `components.video_processor`. Don't create new instances inside block `__call__`. -5. **Using `InputParam.template()` / `OutputParam.template()` when semantics don't match.** Templates carry predefined descriptions — e.g. the `"latents"` output template means "Denoised latents". Don't use it for initial noisy latents from a prepare-latents step. Use a plain `InputParam(...)` / `OutputParam(...)` with an accurate description instead. +5. **Using `InputParam.template()` / `OutputParam.template()` when semantics don't match.** Templates carry predefined descriptions — e.g. the `"latents"` output template means "Denoised latents". Don't use it as is for initial noisy latents from a prepare-latents step: keep the template for the name and type hint and pass `description=` (`OutputParam.template("latents", description="Initial noisy latents")`), or use a plain `InputParam(...)` / `OutputParam(...)`. 6. **`required=True` on an input that carries a default.** An input cannot be both required and have a default. When that happens, `InputParam` keeps the default, treats the input as optional and logs a warning. This is likely to happen when you reuse a template for its type hint and description but make it required: `InputParam.template("num_inference_steps", required=True)` stays an optional input that defaults to 50, plus a warning. Decide which one you mean: drop `required=True` to keep the template default without the warning, or pass `default=None` explicitly to make the input required. @@ -381,6 +378,8 @@ ComponentSpec( 12. **Helpers taking `components`.** Prefer explicit args, for example, `get_xx_prompt_embeds(text_encoder, tokenizer, prompt)` instead of `get_xx_prompt_embeds(components, prompt)`. Helpers and block methods should take only what they use so dependencies are visible and they can run without an assembled pipeline. Leave the full `components` object to `__call__`, which unpacks what its helpers need. See `get_qwen_prompt_embeds` in `qwenimage/encoders.py`. +13. **An override declaration that restates less than the derived entry.** `input_overrides = {"num_inference_steps": InputParam("num_inference_steps", default=10)}` replaces the derived entry whole, so the type hint and description the sub-block declared are gone. Build the override from the template, `InputParam.template("num_inference_steps", default=10)`, or restate everything. + ## Conversion checklist - [ ] Read original pipeline's `__call__` end-to-end, map stages diff --git a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md index d5d3d53e0412..3889dd489f8b 100644 --- a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md +++ b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md @@ -110,18 +110,7 @@ class ImageProcessingStep(SequentialPipelineBlocks): " - Encodes the image into latent space." ) ``` - -When you create a [`~modular_pipelines.SequentialPipelineBlocks`], properties like `inputs`, `intermediate_outputs`, and `expected_components` are automatically aggregated from the sub-blocks, so there is no need to define them again. - -There are a few properties you should set: - -- `description`: We recommend adding a description for the assembled block to explain what the combined step does. -- `model_name`: This is automatically derived from the sub-blocks but isn't always correct, so you may need to override it. -- `outputs`: By default this is the same as `intermediate_outputs`, but you can manually set it to control which values appear in the doc. This is useful for showing only the final outputs instead of all intermediate values. - -These properties, together with the aggregated `inputs`, `intermediate_outputs`, and `expected_components`, are used to automatically generate the `doc` property. - -The aggregated declarations come from the sub-blocks, and the sequence cannot add or remove any of them. It can redefine how one is declared, though, with `input_overrides`, `intermediate_output_overrides`, `component_overrides` and `config_overrides`. You can reuse the same sub-blocks with another default for an input, another type hint or description, or other repo info for a component. Each is a `{name: declaration}` dict. The name must already be in the aggregated list, and the declaration replaces the aggregated entry whole. +When you create a [`~modular_pipelines.SequentialPipelineBlocks`], `inputs`, `intermediate_outputs`, `expected_components` and `expected_configs` are aggregated from the sub-blocks, so there is no need to define them again. The sequence cannot add or remove any of them, but it can redefine how one is declared with `input_overrides`, `intermediate_output_overrides`, `component_overrides` and `config_overrides`. That lets you reuse the same sub-blocks with another default for an input, another type hint or description, or other repo info for a component. Each is a `{name: declaration}` dict: the name must already be in the aggregated list, and the declaration takes the aggregated entry's place. ```py class ImageProcessingStep(SequentialPipelineBlocks): @@ -139,14 +128,12 @@ class ImageProcessingStep(SequentialPipelineBlocks): } ``` -When two sub-blocks declare different defaults for the same input, the sequence keeps the first one and warns; an `input_overrides` entry explicitly states the default the whole sequence uses. When the input belongs to a conditional sub-block whose branches declare different defaults, the override gives the sequence one value for every workflow instead of the per-branch ones. - +When two sub-blocks declare different defaults for the same input, the sequence keeps the first one and warns; an `input_overrides` entry states the default the whole sequence uses. When the input belongs to a conditional sub-block whose branches declare different defaults, the override gives the sequence one value for every workflow instead of the per-branch ones. -Print the `ImageProcessingStep` block to inspect its sub-blocks, and use `doc` for a full summary of the block's inputs, outputs, and components. +A few properties you set yourself: +- `description`: We recommend adding a description for the assembled block to explain what the combined step does. +- `model_name`: This is automatically derived from the sub-blocks but isn't always correct, so you may need to override it. +- `outputs`: By default this is the same as `intermediate_outputs`. Set it to list only the values the step is for, so the doc shows the final outputs instead of every intermediate value. Everything stays in the pipeline state either way. -```py -blocks = ImageProcessingStep() -print(blocks) -print(blocks.doc) -``` \ No newline at end of file +These properties, together with the aggregated `inputs`, `intermediate_outputs`, and `expected_components`, are used to automatically generate the `doc` property. \ No newline at end of file From b67848785f65c688c08f2f31cd1d531cd346ad59 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Fri, 13 Nov 2026 06:33:33 +0100 Subject: [PATCH 08/33] [modular] HunyuanVideo 1.5: expand text embeddings in the input step, not the encoder The text encoder repeated the embeddings by `num_videos_per_prompt` and passed the prompt count along as `batch_size` on the state, so the pipeline listed `batch_size` as an input. The encoder now emits one row per prompt and `HunyuanVideo15TextInputStep` derives `batch_size` and expands the embeddings, like the other pipelines. Co-Authored-By: Claude Fable 5.1 --- .../hunyuan_video1_5/before_denoise.py | 85 +++++++++++- .../hunyuan_video1_5/encoders.py | 23 ---- .../modular_blocks_hunyuan_video1_5.py | 122 ++++++++---------- 3 files changed, 133 insertions(+), 97 deletions(-) diff --git a/src/diffusers/modular_pipelines/hunyuan_video1_5/before_denoise.py b/src/diffusers/modular_pipelines/hunyuan_video1_5/before_denoise.py index f478c25b82e3..0d08ba8fedc9 100644 --- a/src/diffusers/modular_pipelines/hunyuan_video1_5/before_denoise.py +++ b/src/diffusers/modular_pipelines/hunyuan_video1_5/before_denoise.py @@ -96,19 +96,78 @@ class HunyuanVideo15TextInputStep(ModularPipelineBlocks): @property def description(self) -> str: - return "Input processing step that determines batch_size" + return ( + "Input processing step that:\n" + " 1. Determines `batch_size` based on `prompt_embeds`\n" + " 2. Expands the text embeddings and their masks to `batch_size * num_videos_per_prompt`\n\n" + "This block should be placed after all encoder steps to process the text embeddings before they are used in " + "subsequent pipeline steps." + ) @property def inputs(self) -> list[InputParam]: return [ + InputParam.template("num_images_per_prompt", name="num_videos_per_prompt"), InputParam.template("prompt_embeds"), - InputParam.template("batch_size", default=None), + InputParam.template("prompt_embeds_mask"), + InputParam.template("negative_prompt_embeds"), + InputParam.template("negative_prompt_embeds_mask"), + InputParam( + "prompt_embeds_2", + required=True, + type_hint=torch.Tensor, + description="ByT5 glyph-text embeddings. Can be generated from text_encoder step.", + ), + InputParam( + "prompt_embeds_mask_2", + required=True, + type_hint=torch.Tensor, + description="Attention mask for the ByT5 glyph-text embeddings. Can be generated from text_encoder step.", + ), + InputParam( + "negative_prompt_embeds_2", + type_hint=torch.Tensor, + description="ByT5 glyph-text negative embeddings. Can be generated from text_encoder step.", + ), + InputParam( + "negative_prompt_embeds_mask_2", + type_hint=torch.Tensor, + description="Attention mask for the ByT5 glyph-text negative embeddings. Can be generated from text_encoder step.", + ), ] @property def intermediate_outputs(self) -> list[OutputParam]: return [ - OutputParam("batch_size", type_hint=int), + OutputParam("batch_size", type_hint=int, description="Number of prompts"), + OutputParam.template("prompt_embeds", note="batch-expanded"), + OutputParam.template("prompt_embeds_mask", note="batch-expanded"), + OutputParam.template("negative_prompt_embeds", note="batch-expanded"), + OutputParam.template("negative_prompt_embeds_mask", note="batch-expanded"), + OutputParam( + "prompt_embeds_2", + type_hint=torch.Tensor, + kwargs_type="denoiser_input_fields", + description="ByT5 glyph-text embeddings used as a second conditioning stream for the transformer. (batch-expanded)", + ), + OutputParam( + "prompt_embeds_mask_2", + type_hint=torch.Tensor, + kwargs_type="denoiser_input_fields", + description="Attention mask for the ByT5 glyph-text embeddings. (batch-expanded)", + ), + OutputParam( + "negative_prompt_embeds_2", + type_hint=torch.Tensor, + kwargs_type="denoiser_input_fields", + description="ByT5 glyph-text negative embeddings for classifier-free guidance. (batch-expanded)", + ), + OutputParam( + "negative_prompt_embeds_mask_2", + type_hint=torch.Tensor, + kwargs_type="denoiser_input_fields", + description="Attention mask for the ByT5 glyph-text negative embeddings. (batch-expanded)", + ), ] @torch.no_grad() @@ -116,7 +175,25 @@ def __call__( self, components: HunyuanVideo15ModularPipeline, state: PipelineState ) -> tuple[HunyuanVideo15ModularPipeline, PipelineState]: block_state = self.get_block_state(state) - block_state.batch_size = getattr(block_state, "batch_size", None) or block_state.prompt_embeds.shape[0] + block_state.batch_size = block_state.prompt_embeds.shape[0] + + # the text encoder emits one row per prompt; expand every embedding and mask to batch_size * num_videos_per_prompt + for name in ( + "prompt_embeds", + "prompt_embeds_mask", + "negative_prompt_embeds", + "negative_prompt_embeds_mask", + "prompt_embeds_2", + "prompt_embeds_mask_2", + "negative_prompt_embeds_2", + "negative_prompt_embeds_mask_2", + ): + value = getattr(block_state, name) + if value is None: + continue + value = value.repeat_interleave(block_state.num_videos_per_prompt, dim=0) + setattr(block_state, name, value) + self.set_block_state(state, block_state) return components, state diff --git a/src/diffusers/modular_pipelines/hunyuan_video1_5/encoders.py b/src/diffusers/modular_pipelines/hunyuan_video1_5/encoders.py index 11f511d7e4e6..83cd7105376c 100644 --- a/src/diffusers/modular_pipelines/hunyuan_video1_5/encoders.py +++ b/src/diffusers/modular_pipelines/hunyuan_video1_5/encoders.py @@ -165,7 +165,6 @@ def inputs(self) -> list[InputParam]: return [ InputParam.template("prompt", required=False), InputParam.template("negative_prompt"), - InputParam.template("num_images_per_prompt", name="num_videos_per_prompt"), ] @property @@ -208,7 +207,6 @@ def encode_prompt( device=None, dtype=None, batch_size=1, - num_videos_per_prompt=1, ): device = device or components._execution_device dtype = dtype or components.text_encoder.dtype @@ -235,22 +233,6 @@ def encode_prompt( tokenizer_max_length=components.tokenizer_2_max_length, ) - _, seq_len, _ = prompt_embeds.shape - prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1).view( - batch_size * num_videos_per_prompt, seq_len, -1 - ) - prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_videos_per_prompt, 1).view( - batch_size * num_videos_per_prompt, seq_len - ) - - _, seq_len_2, _ = prompt_embeds_2.shape - prompt_embeds_2 = prompt_embeds_2.repeat(1, num_videos_per_prompt, 1).view( - batch_size * num_videos_per_prompt, seq_len_2, -1 - ) - prompt_embeds_mask_2 = prompt_embeds_mask_2.repeat(1, num_videos_per_prompt, 1).view( - batch_size * num_videos_per_prompt, seq_len_2 - ) - prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) prompt_embeds_mask = prompt_embeds_mask.to(dtype=dtype, device=device) prompt_embeds_2 = prompt_embeds_2.to(dtype=dtype, device=device) @@ -268,7 +250,6 @@ def __call__( prompt = block_state.prompt negative_prompt = block_state.negative_prompt - num_videos_per_prompt = block_state.num_videos_per_prompt if prompt is not None and isinstance(prompt, str): batch_size = 1 @@ -288,7 +269,6 @@ def __call__( device=device, dtype=dtype, batch_size=batch_size, - num_videos_per_prompt=num_videos_per_prompt, ) if components.requires_unconditional_embeds: @@ -303,11 +283,8 @@ def __call__( device=device, dtype=dtype, batch_size=batch_size, - num_videos_per_prompt=num_videos_per_prompt, ) - state.set("batch_size", batch_size) - self.set_block_state(state, block_state) return components, state diff --git a/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py b/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py index 0fb2f085a544..f073b22c0b40 100644 --- a/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py +++ b/src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py @@ -43,11 +43,24 @@ class HunyuanVideo15CoreDenoiseStep(SequentialPipelineBlocks): video_processor (`HunyuanVideo15ImageProcessor`) guider (`ClassifierFreeGuidance`) Inputs: + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. prompt_embeds (`Tensor`): text embeddings used to guide the image generation. Can be generated from text_encoder step. - batch_size (`int`, *optional*): - Number of prompts, the final batch size of model inputs should be batch_size * num_images_per_prompt. Can - be generated in input step. + prompt_embeds_mask (`Tensor`): + mask for the text embeddings. Can be generated from text_encoder step. + negative_prompt_embeds (`Tensor`, *optional*): + negative text embeddings used to guide the image generation. Can be generated from text_encoder step. + negative_prompt_embeds_mask (`Tensor`, *optional*): + mask for the negative text embeddings. Can be generated from text_encoder step. + prompt_embeds_2 (`Tensor`): + ByT5 glyph-text embeddings. Can be generated from text_encoder step. + prompt_embeds_mask_2 (`Tensor`): + Attention mask for the ByT5 glyph-text embeddings. Can be generated from text_encoder step. + negative_prompt_embeds_2 (`Tensor`, *optional*): + ByT5 glyph-text negative embeddings. Can be generated from text_encoder step. + negative_prompt_embeds_mask_2 (`Tensor`, *optional*): + Attention mask for the ByT5 glyph-text negative embeddings. Can be generated from text_encoder step. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): @@ -60,26 +73,10 @@ class HunyuanVideo15CoreDenoiseStep(SequentialPipelineBlocks): Number of video frames to generate. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. - num_videos_per_prompt (`int`, *optional*, defaults to 1): - The number of images to generate per prompt. generator (`Generator`, *optional*): Torch generator for deterministic generation. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. - negative_prompt_embeds (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds' field fed into the guider. - prompt_embeds_mask (`Tensor`): - Positive branch of the 'prompt_embeds_mask' field fed into the guider. - negative_prompt_embeds_mask (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds_mask' field fed into the guider. - prompt_embeds_2 (`Tensor`): - Positive branch of the 'prompt_embeds_2' field fed into the guider. - negative_prompt_embeds_2 (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds_2' field fed into the guider. - prompt_embeds_mask_2 (`Tensor`): - Positive branch of the 'prompt_embeds_mask_2' field fed into the guider. - negative_prompt_embeds_mask_2 (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds_mask_2' field fed into the guider. Outputs: latents (`Tensor`): @@ -122,9 +119,6 @@ class HunyuanVideo15Blocks(SequentialPipelineBlocks): The prompt or prompts not to guide the image generation. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. - batch_size (`int`, *optional*): - Number of prompts, the final batch size of model inputs should be batch_size * num_images_per_prompt. Can - be generated in input step. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): @@ -176,11 +170,24 @@ class HunyuanVideo15Image2VideoCoreDenoiseStep(SequentialPipelineBlocks): video_processor (`HunyuanVideo15ImageProcessor`) guider (`ClassifierFreeGuidance`) Inputs: + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. prompt_embeds (`Tensor`): text embeddings used to guide the image generation. Can be generated from text_encoder step. - batch_size (`int`, *optional*): - Number of prompts, the final batch size of model inputs should be batch_size * num_images_per_prompt. Can - be generated in input step. + prompt_embeds_mask (`Tensor`): + mask for the text embeddings. Can be generated from text_encoder step. + negative_prompt_embeds (`Tensor`, *optional*): + negative text embeddings used to guide the image generation. Can be generated from text_encoder step. + negative_prompt_embeds_mask (`Tensor`, *optional*): + mask for the negative text embeddings. Can be generated from text_encoder step. + prompt_embeds_2 (`Tensor`): + ByT5 glyph-text embeddings. Can be generated from text_encoder step. + prompt_embeds_mask_2 (`Tensor`): + Attention mask for the ByT5 glyph-text embeddings. Can be generated from text_encoder step. + negative_prompt_embeds_2 (`Tensor`, *optional*): + ByT5 glyph-text negative embeddings. Can be generated from text_encoder step. + negative_prompt_embeds_mask_2 (`Tensor`, *optional*): + Attention mask for the ByT5 glyph-text negative embeddings. Can be generated from text_encoder step. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): @@ -193,28 +200,12 @@ class HunyuanVideo15Image2VideoCoreDenoiseStep(SequentialPipelineBlocks): Number of video frames to generate. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. - num_videos_per_prompt (`int`, *optional*, defaults to 1): - The number of images to generate per prompt. generator (`Generator`, *optional*): Torch generator for deterministic generation. image_latents (`Tensor`): Pre-encoded image latents from the VAE encoder step, used as conditioning for I2V. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. - negative_prompt_embeds (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds' field fed into the guider. - prompt_embeds_mask (`Tensor`): - Positive branch of the 'prompt_embeds_mask' field fed into the guider. - negative_prompt_embeds_mask (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds_mask' field fed into the guider. - prompt_embeds_2 (`Tensor`): - Positive branch of the 'prompt_embeds_2' field fed into the guider. - negative_prompt_embeds_2 (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds_2' field fed into the guider. - prompt_embeds_mask_2 (`Tensor`): - Positive branch of the 'prompt_embeds_mask_2' field fed into the guider. - negative_prompt_embeds_mask_2 (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds_mask_2' field fed into the guider. Outputs: latents (`Tensor`): @@ -330,11 +321,24 @@ class HunyuanVideo15AutoCoreDenoiseStep(AutoPipelineBlocks): video_processor (`HunyuanVideo15ImageProcessor`) guider (`ClassifierFreeGuidance`) Inputs: + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. prompt_embeds (`Tensor`): text embeddings used to guide the image generation. Can be generated from text_encoder step. - batch_size (`int`, *optional*): - Number of prompts, the final batch size of model inputs should be batch_size * num_images_per_prompt. Can - be generated in input step. + prompt_embeds_mask (`Tensor`): + mask for the text embeddings. Can be generated from text_encoder step. + negative_prompt_embeds (`Tensor`, *optional*): + negative text embeddings used to guide the image generation. Can be generated from text_encoder step. + negative_prompt_embeds_mask (`Tensor`, *optional*): + mask for the negative text embeddings. Can be generated from text_encoder step. + prompt_embeds_2 (`Tensor`): + ByT5 glyph-text embeddings. Can be generated from text_encoder step. + prompt_embeds_mask_2 (`Tensor`): + Attention mask for the ByT5 glyph-text embeddings. Can be generated from text_encoder step. + negative_prompt_embeds_2 (`Tensor`, *optional*): + ByT5 glyph-text negative embeddings. Can be generated from text_encoder step. + negative_prompt_embeds_mask_2 (`Tensor`, *optional*): + Attention mask for the ByT5 glyph-text negative embeddings. Can be generated from text_encoder step. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): @@ -347,28 +351,12 @@ class HunyuanVideo15AutoCoreDenoiseStep(AutoPipelineBlocks): Number of video frames to generate. latents (`Tensor`, *optional*): Pre-generated noisy latents for image generation. - num_videos_per_prompt (`int`, *optional*, defaults to 1): - The number of images to generate per prompt. generator (`Generator`, *optional*): Torch generator for deterministic generation. image_latents (`Tensor`, *optional*, depends on the workflow): Pre-encoded image latents from the VAE encoder step, used as conditioning for I2V. attention_kwargs (`dict`, *optional*): Additional kwargs for attention processors. - negative_prompt_embeds (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds' field fed into the guider. - prompt_embeds_mask (`Tensor`): - Positive branch of the 'prompt_embeds_mask' field fed into the guider. - negative_prompt_embeds_mask (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds_mask' field fed into the guider. - prompt_embeds_2 (`Tensor`): - Positive branch of the 'prompt_embeds_2' field fed into the guider. - negative_prompt_embeds_2 (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds_2' field fed into the guider. - prompt_embeds_mask_2 (`Tensor`): - Positive branch of the 'prompt_embeds_mask_2' field fed into the guider. - negative_prompt_embeds_mask_2 (`Tensor`, *optional*): - Negative branch of the 'negative_prompt_embeds_mask_2' field fed into the guider. Outputs: latents (`Tensor`): @@ -410,17 +398,14 @@ class HunyuanVideo15AutoBlocks(SequentialPipelineBlocks): The prompt or prompts to guide image generation. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. - num_videos_per_prompt (`int`, *optional*, defaults to 1): - The number of images to generate per prompt. image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - batch_size (`int`, *optional*): - Number of prompts, the final batch size of model inputs should be batch_size * num_images_per_prompt. Can - be generated in input step. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): @@ -483,17 +468,14 @@ class HunyuanVideo15Image2VideoBlocks(SequentialPipelineBlocks): The prompt or prompts to guide image generation. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. - num_videos_per_prompt (`int`, *optional*, defaults to 1): - The number of images to generate per prompt. image (`Image | list`, *optional*, depends on the workflow): Reference image(s) for denoising. Can be a single image or list of images. height (`int`, *optional*): The height in pixels of the generated image. width (`int`, *optional*): The width in pixels of the generated image. - batch_size (`int`, *optional*): - Number of prompts, the final batch size of model inputs should be batch_size * num_images_per_prompt. Can - be generated in input step. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): From b7e3831b38866520bc572c733640c69a74c224c8 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Fri, 13 Nov 2026 07:57:49 +0100 Subject: [PATCH 09/33] [docs] modular: tighten the composite inputs wording Co-Authored-By: Claude Fable 5.1 --- .ai/references/modular.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ai/references/modular.md b/.ai/references/modular.md index 55f97284b18c..4b2be31cae2b 100644 --- a/.ai/references/modular.md +++ b/.ai/references/modular.md @@ -314,11 +314,11 @@ class DistilledCoreDenoiseStep(SequentialPipelineBlocks): How the composite derives a shared input, and when you need an override: -- In a conditional block, the sub-blocks may declare different defaults for the same input, and each is respected: e.g. for `strength`, the img2img branch defaults to 0.3 and the inpaint branch to 0.9999, the docstring of the conditional block says "depends on the workflow", and at runtime the branch that runs fills in its own. An `input_overrides` entry on the conditional block will gives every branch the same default and cannot set one per branch. To change a single branch's default, override the input on that sub-block. +- In a conditional block, the sub-blocks may declare different defaults for the same input, and each is respected: e.g. for `strength`, the img2img branch defaults to 0.3 and the inpaint branch to 0.9999, the docstring of the conditional block says "depends on the workflow", and at runtime the branch that runs fills in its own. An `input_overrides` entry on the conditional block gives every branch the same default and cannot set one per branch. To change a single branch's default, override the input on that sub-block. - In a sequential block, the first sub-block that declares a default sets it for every sub-block after it. When a later sub-block declares a different default, the sequence keeps the first one and warns that the default is ambiguous. State it explicitly: declare the input in `input_overrides` on the composite. -- The `required` flag the composite reports is its own view, whether the caller has to pass the value. A sub-block may require an input that an earlier sub-block defaults or may produce; the composite then lists it as optional. A conditional block lists an input as required only when every branch requires it and the block always runs. The sub-blocks' own declarations are untouched and still checked at runtime in `get_block_state`. +- The `required` flag the composite reports is its own view, whether the caller has to pass the value. In a sequential block, a sub-block may require an input that an earlier sub-block defaults or may produce; the composite then lists it as optional. In a conditional block, an input is required only when every branch requires it and the block always runs. The sub-blocks' own declarations are untouched and still checked at runtime in `get_block_state`. -**`outputs` is what the assembled block presents as its product.** By default it is every intermediate output. Override the property to list only what the step is for, so the docstring shows the product instead of every internal intermediate (Wan-Animate-2's core denoise lists only `segment_frames`). It changes the docstring and `output_names` only: everything stays in the pipeline state, and `pipe(..., output="latents")` still fetches any intermediate. +**`outputs` is what the assembled block presents as its product.** By default it is every intermediate output. Override the property to list only what the step is for, so the docstring shows the product instead of every internal intermediate (Wan-Animate-2's core denoise lists only `segment_frames`). It changes the docstring and `output_names` only: everything stays in the pipeline state, and `pipe(..., output=)` still fetches any intermediate, listed or not. ## ComponentSpec patterns From 3a9f1cafe288f2712961a3782cf6dfdbc8775a97 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sat, 14 Nov 2026 00:15:02 +0100 Subject: [PATCH 10/33] [modular] remove `intermediate_output_overrides` No block uses it, and a composite already defines `outputs` to choose which outputs it lists and how it describes them. Co-Authored-By: Claude Fable 5.2 --- .ai/references/modular.md | 2 +- .../sequential_pipeline_blocks.md | 2 +- .../modular_pipelines/modular_pipeline.py | 43 ++----------------- .../modular_pipeline_utils.py | 2 +- .../test_sequential_pipeline_blocks.py | 6 +-- 5 files changed, 7 insertions(+), 48 deletions(-) diff --git a/.ai/references/modular.md b/.ai/references/modular.md index 4b2be31cae2b..f1e240b35748 100644 --- a/.ai/references/modular.md +++ b/.ai/references/modular.md @@ -300,7 +300,7 @@ The other two ways to declare an input: (1) `required=True` when the block cannot work without the value, so the caller has to pass it; the block raises `ValueError("Required input 'x' is missing")` when it is not passed or `None`. (2) Optional with no default when the block works with `None`. Most often that is the computed case above, and the block should then also declare the value as an output. -**An assembled block derives its declarations from its sub-blocks, and can redefine them with `*_overrides`.** Inputs, outputs, components and configs are declared on leaf blocks. A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its own from its sub-blocks. It cannot add or remove any of them, they are part of the blocks' logic, but it can redefine how one is declared: the default or description of an input, the description of an output, the repo and loading info of a component, the value of a config. You set these as class attributes on the composite, or pass them to `from_blocks_dict`: `input_overrides`, `intermediate_output_overrides`, `component_overrides` and `config_overrides`, each a `{name: declaration}` dict. The name must be one the composite already derives, and the declaration takes the derived entry's place (gotcha #13). +**An assembled block derives its declarations from its sub-blocks, and can redefine them with `*_overrides`.** Inputs, outputs, components and configs are declared on leaf blocks. A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its own from its sub-blocks. It cannot add or remove any of them, they are part of the blocks' logic, but it can redefine how one is declared: the default or description of an input, the repo and loading info of a component, the value of a config. You set these as class attributes on the composite, or pass them to `from_blocks_dict`: `input_overrides`, `component_overrides` and `config_overrides`, each a `{name: declaration}` dict. The name must be one the composite already derives, and the declaration takes the derived entry's place (gotcha #13). `intermediate_outputs` has no override: a composite can define the `outputs` property to choose the ones it lists and how it describes them. ```python class DistilledCoreDenoiseStep(SequentialPipelineBlocks): diff --git a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md index 3889dd489f8b..39b1ffeb8b5a 100644 --- a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md +++ b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md @@ -110,7 +110,7 @@ class ImageProcessingStep(SequentialPipelineBlocks): " - Encodes the image into latent space." ) ``` -When you create a [`~modular_pipelines.SequentialPipelineBlocks`], `inputs`, `intermediate_outputs`, `expected_components` and `expected_configs` are aggregated from the sub-blocks, so there is no need to define them again. The sequence cannot add or remove any of them, but it can redefine how one is declared with `input_overrides`, `intermediate_output_overrides`, `component_overrides` and `config_overrides`. That lets you reuse the same sub-blocks with another default for an input, another type hint or description, or other repo info for a component. Each is a `{name: declaration}` dict: the name must already be in the aggregated list, and the declaration takes the aggregated entry's place. +When you create a [`~modular_pipelines.SequentialPipelineBlocks`], `inputs`, `intermediate_outputs`, `expected_components` and `expected_configs` are aggregated from the sub-blocks, so there is no need to define them again. The sequence cannot add or remove any of them, but it can redefine how an input, a component or a config is declared with `input_overrides`, `component_overrides` and `config_overrides`. That lets you reuse the same sub-blocks with another default for an input, or other repo info for a component. Each is a `{name: declaration}` dict: the name must already be in the aggregated list, and the declaration takes the aggregated entry's place. ```py class ImageProcessingStep(SequentialPipelineBlocks): diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index e011e64fa5f3..9fca037c45b2 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -705,9 +705,6 @@ class ConditionalPipelineBlocks(ModularPipelineBlocks): `required`, description. When the sub-blocks declare different defaults for the same input, each one keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with one value. - intermediate_output_overrides: the composite block's own declarations for intermediate outputs it derives - from its sub-blocks, as `{name: OutputParam}`, e.g. - `{"latents": OutputParam("latents", description="Denoised latents.")}`. The name must be a derived output. component_overrides: the composite block's own specs for components it derives from its sub-blocks, as `{name: ComponentSpec}`, e.g. `{"text_encoder": ComponentSpec("text_encoder", T5EncoderModel)}`. The name must be a derived component. @@ -721,7 +718,6 @@ class ConditionalPipelineBlocks(ModularPipelineBlocks): block_trigger_inputs = [] default_block_name = None input_overrides: dict[str, InputParam] = {} - intermediate_output_overrides: dict[str, OutputParam] = {} component_overrides: dict[str, ComponentSpec] = {} config_overrides: dict[str, ConfigSpec] = {} @@ -851,12 +847,7 @@ def inputs(self) -> list[tuple[str, Any]]: def intermediate_outputs(self) -> list[str]: named_outputs = [(name, block.intermediate_outputs) for name, block in self.sub_blocks.items()] combined_outputs = combine_outputs(*named_outputs) - return override_attribute( - combined_outputs, - self.intermediate_output_overrides, - "intermediate_output_overrides", - self.__class__.__name__, - ) + return combined_outputs @property def outputs(self) -> list[str]: @@ -1133,10 +1124,6 @@ class SequentialPipelineBlocks(ModularPipelineBlocks): `required`, description. When the sub-blocks declare different defaults for the same input, each one keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with one value. Can also be passed to `from_blocks_dict`. - intermediate_output_overrides: the composite block's own declarations for intermediate outputs it derives - from its sub-blocks, as `{name: OutputParam}`, e.g. - `{"latents": OutputParam("latents", description="Denoised latents.")}`. The name must be a derived output. - Can also be passed to `from_blocks_dict`. component_overrides: the composite block's own specs for components it derives from its sub-blocks, as `{name: ComponentSpec}`, e.g. `{"text_encoder": ComponentSpec("text_encoder", T5EncoderModel)}`. The name must be a derived component. Can also be passed to `from_blocks_dict`. @@ -1148,7 +1135,6 @@ class SequentialPipelineBlocks(ModularPipelineBlocks): block_classes = [] block_names = [] input_overrides: dict[str, InputParam] = {} - intermediate_output_overrides: dict[str, OutputParam] = {} component_overrides: dict[str, ComponentSpec] = {} config_overrides: dict[str, ConfigSpec] = {} @@ -1216,7 +1202,6 @@ def from_blocks_dict( blocks_dict: dict[str, Any], description: str | None = None, input_overrides: dict[str, InputParam] | None = None, - intermediate_output_overrides: dict[str, OutputParam] | None = None, component_overrides: dict[str, ComponentSpec] | None = None, config_overrides: dict[str, ConfigSpec] | None = None, ) -> "SequentialPipelineBlocks": @@ -1226,8 +1211,6 @@ def from_blocks_dict( blocks_dict: Dictionary mapping block names to block classes or instances description: Description of the new block input_overrides: the composite's own declarations replacing derived `inputs`, as `{name: InputParam}` - intermediate_output_overrides: the composite's own declarations replacing derived - `intermediate_outputs`, as `{name: OutputParam}` component_overrides: the composite's own specs replacing derived `expected_components`, as `{name: ComponentSpec}` config_overrides: the composite's own specs replacing derived `expected_configs`, as `{name: ConfigSpec}` @@ -1253,7 +1236,6 @@ def from_blocks_dict( instance.description = description for name, value in ( ("input_overrides", input_overrides), - ("intermediate_output_overrides", intermediate_output_overrides), ("component_overrides", component_overrides), ("config_overrides", config_overrides), ): @@ -1370,12 +1352,7 @@ def inputs(self) -> list[tuple[str, Any]]: def intermediate_outputs(self) -> list[str]: named_outputs = [(name, block.intermediate_outputs) for name, block in self.sub_blocks.items()] combined_outputs = combine_outputs(*named_outputs) - return override_attribute( - combined_outputs, - self.intermediate_output_overrides, - "intermediate_output_overrides", - self.__class__.__name__, - ) + return combined_outputs # `outputs` is what `doc` and the model card list as the block's outputs. By default that is every intermediate # output; a composite overrides it to show only what the step is for, e.g. just `latents` for the `denoise` block. @@ -1584,10 +1561,6 @@ class LoopSequentialPipelineBlocks(ModularPipelineBlocks): `required`, description. When the sub-blocks declare different defaults for the same input, each one keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with one value. Can also be passed to `from_blocks_dict`. - intermediate_output_overrides: the composite block's own declarations for intermediate outputs it derives - from its sub-blocks, as `{name: OutputParam}`, e.g. - `{"latents": OutputParam("latents", description="Denoised latents.")}`. The name must be a derived output. - Can also be passed to `from_blocks_dict`. component_overrides: the composite block's own specs for components it derives from its sub-blocks, as `{name: ComponentSpec}`, e.g. `{"text_encoder": ComponentSpec("text_encoder", T5EncoderModel)}`. The name must be a derived component. Can also be passed to `from_blocks_dict`. @@ -1600,7 +1573,6 @@ class LoopSequentialPipelineBlocks(ModularPipelineBlocks): block_classes = [] block_names = [] input_overrides: dict[str, InputParam] = {} - intermediate_output_overrides: dict[str, OutputParam] = {} component_overrides: dict[str, ComponentSpec] = {} config_overrides: dict[str, ConfigSpec] = {} @@ -1712,12 +1684,7 @@ def intermediate_outputs(self) -> list[str]: for output in self.loop_intermediate_outputs: if output.name not in {output.name for output in combined_outputs}: combined_outputs.append(output) - return override_attribute( - combined_outputs, - self.intermediate_output_overrides, - "intermediate_output_overrides", - self.__class__.__name__, - ) + return combined_outputs # YiYi TODO: this need to be thought about more @property @@ -1755,7 +1722,6 @@ def from_blocks_dict( cls, blocks_dict: dict[str, Any], input_overrides: dict[str, InputParam] | None = None, - intermediate_output_overrides: dict[str, OutputParam] | None = None, component_overrides: dict[str, ComponentSpec] | None = None, config_overrides: dict[str, ConfigSpec] | None = None, ) -> "LoopSequentialPipelineBlocks": @@ -1765,8 +1731,6 @@ def from_blocks_dict( Args: blocks_dict: Dictionary mapping block names to block instances input_overrides: the composite's own declarations replacing derived `inputs`, as `{name: InputParam}` - intermediate_output_overrides: the composite's own declarations replacing derived - `intermediate_outputs`, as `{name: OutputParam}` component_overrides: the composite's own specs replacing derived `expected_components`, as `{name: ComponentSpec}` config_overrides: the composite's own specs replacing derived `expected_configs`, as `{name: ConfigSpec}` @@ -1789,7 +1753,6 @@ def from_blocks_dict( instance.sub_blocks = blocks_dict for name, value in ( ("input_overrides", input_overrides), - ("intermediate_output_overrides", intermediate_output_overrides), ("component_overrides", component_overrides), ("config_overrides", config_overrides), ): diff --git a/src/diffusers/modular_pipelines/modular_pipeline_utils.py b/src/diffusers/modular_pipelines/modular_pipeline_utils.py index f9e09afa48e9..665f170101e9 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline_utils.py +++ b/src/diffusers/modular_pipelines/modular_pipeline_utils.py @@ -1131,7 +1131,7 @@ def override_attribute(items: list, overrides: dict[str, Any], attr_name: str, b Apply a composite block's `*_overrides` to the list it derived from its sub-blocks. Args: - items: the derived `InputParam`, `OutputParam`, `ComponentSpec` or `ConfigSpec` list + items: the derived `InputParam`, `ComponentSpec` or `ConfigSpec` list overrides: `{name: declaration}`; every name must be in `items`, and the declaration, an object of the same class as the entry it replaces, takes that entry's place whole attr_name: the overrides attribute being applied, e.g. `"input_overrides"`, for error messages diff --git a/tests/modular_pipelines/test_sequential_pipeline_blocks.py b/tests/modular_pipelines/test_sequential_pipeline_blocks.py index e061bedf7737..1349db3c6966 100644 --- a/tests/modular_pipelines/test_sequential_pipeline_blocks.py +++ b/tests/modular_pipelines/test_sequential_pipeline_blocks.py @@ -779,13 +779,11 @@ def test_override_does_not_mutate_the_sub_block_declaration(self): assert batch_size.required is True assert batch_size.default is None - def test_output_component_and_config_overrides(self): + def test_component_and_config_overrides(self): blocks = self._sequential( - intermediate_output_overrides={"latents": OutputParam("latents", description="final latents")}, component_overrides={"unet": ComponentSpec("unet", description="any UNet2DConditionModel")}, config_overrides={"scale": ConfigSpec("scale", default=16)}, ) - assert [out.description for out in blocks.intermediate_outputs if out.name == "latents"] == ["final latents"] assert [c.description for c in blocks.expected_components if c.name == "unet"] == ["any UNet2DConditionModel"] assert [c.default for c in blocks.expected_configs if c.name == "scale"] == [16] @@ -806,11 +804,9 @@ def test_from_blocks_dict_accepts_overrides(self): blocks = SequentialPipelineBlocks.from_blocks_dict( {"prepare_latents": PrepareLatentsBlock(), "unet": UNetBlock()}, input_overrides={"batch_size": InputParam("batch_size", default=2)}, - intermediate_output_overrides={"latents": OutputParam("latents", description="final latents")}, config_overrides={"scale": ConfigSpec("scale", default=16)}, ) assert [inp.default for inp in blocks.inputs if inp.name == "batch_size"] == [2] - assert [out.description for out in blocks.intermediate_outputs if out.name == "latents"] == ["final latents"] assert [c.default for c in blocks.expected_configs if c.name == "scale"] == [16] loop = LoopSequentialPipelineBlocks.from_blocks_dict( From 649bc13fae1871f91f9634ab2f0da7cd39aee3de Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sat, 14 Nov 2026 01:07:00 +0100 Subject: [PATCH 11/33] [modular] a skippable block nested in a sub-block still only maybe produces its outputs The sequential walk looked one level into a conditional sub-block. A composite block now marks an output that is only produced on some runs, so the walk gets the right answer at any depth. Co-Authored-By: Claude Fable 5.2 --- .../modular_pipelines/modular_pipeline.py | 38 ++++++++----- .../modular_pipeline_utils.py | 3 + .../test_conditional_pipeline_blocks.py | 57 +++++++++++++++++++ .../test_sequential_pipeline_blocks.py | 55 ++++++++++++++++++ 4 files changed, 139 insertions(+), 14 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 9fca037c45b2..a782e42bbcd3 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -845,9 +845,23 @@ def inputs(self) -> list[tuple[str, Any]]: @property def intermediate_outputs(self) -> list[str]: + # the outputs of each branch, e.g. [("img2img", [, ]), ("text2img", [])] named_outputs = [(name, block.intermediate_outputs) for name, block in self.sub_blocks.items()] + # merged into one list, one entry per name, e.g. [, ] combined_outputs = combine_outputs(*named_outputs) - return combined_outputs + # determine the list of outputs that are always produced + if self.default_block_name is None: + # if the block can be skipped, no outputs are always produced + always = set() + else: + # if the block cannot be skipped, and every branch produces the output, then the output is always produced + # one set of names per branch, e.g. per_branch = [{"latents", "image_latents"}, {"latents"}]: the first set + # is the `img2img` branch, the second the `text2img` branch + per_branch = [{out.name for out in outputs if not out._maybe} for _, outputs in named_outputs] + # e.g. always = {"latents"} as `latents` is present in both branches + always = set.intersection(*per_branch) + # use `replace` so that we don't modify the sub-block's declaration + return [replace(out, _maybe=out.name not in always) for out in combined_outputs] @property def outputs(self) -> list[str]: @@ -1268,7 +1282,7 @@ def _get_inputs(self): inputs = {} # input_name -> InputParam, e.g. {"num_inference_steps": InputParam("num_inference_steps", default=50)} outputs = set() # produced by an earlier block # maybe produced by an earlier block: a ConditionalPipelineBlocks that can be skipped, or whose branches do not - # all produce the output + # all produce the output, at any depth maybe_outputs = set() for block_name, block in self.sub_blocks.items(): for inp in block.inputs: @@ -1328,17 +1342,10 @@ def _get_inputs(self): f"earlier declaration. Declare '{key}' in `input_overrides` on {self.__class__.__name__} " f"if that is not intended." ) - if isinstance(block, ConditionalPipelineBlocks): - # for conditional blocks, - # if the output is produced by every branch and the block itself cannot be skipped, add it to the `outputs` - # otherwise, add it to the `maybe_outputs` - # e.g. `image_encoder` may be skipped, so `image_latents` is only produced if it runs, a maybe_outputs - per_branch = [{out.name for out in sub.intermediate_outputs} for sub in block.sub_blocks.values()] - produced = set.intersection(*per_branch) if block.default_block_name is not None else set() - outputs.update(produced) - maybe_outputs.update(set().union(*per_branch) - produced) - else: - outputs.update(out.name for out in block.intermediate_outputs) + # an output the block always produces goes to `outputs`, one marked `_maybe` (see `OutputParam`) to + # `maybe_outputs`, e.g. `image_encoder` may be skipped, so `image_latents` is only produced if it runs + for out in block.intermediate_outputs: + (maybe_outputs if out._maybe else outputs).add(out.name) return override_attribute( list(inputs.values()), self.input_overrides, "input_overrides", self.__class__.__name__ @@ -1352,7 +1359,10 @@ def inputs(self) -> list[tuple[str, Any]]: def intermediate_outputs(self) -> list[str]: named_outputs = [(name, block.intermediate_outputs) for name, block in self.sub_blocks.items()] combined_outputs = combine_outputs(*named_outputs) - return combined_outputs + # every sub-block runs, so an output is always produced when any sub-block always produces it. A maybe can only + # come from a conditional block nested in a sub-block, see `ConditionalPipelineBlocks.intermediate_outputs` + always = {out.name for _, outputs in named_outputs for out in outputs if not out._maybe} + return [replace(out, _maybe=out.name not in always) for out in combined_outputs] # `outputs` is what `doc` and the model card list as the block's outputs. By default that is every intermediate # output; a composite overrides it to show only what the step is for, e.g. just `latents` for the `denoise` block. diff --git a/src/diffusers/modular_pipelines/modular_pipeline_utils.py b/src/diffusers/modular_pipelines/modular_pipeline_utils.py index 665f170101e9..c09db0f9bd1b 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline_utils.py +++ b/src/diffusers/modular_pipelines/modular_pipeline_utils.py @@ -630,6 +630,9 @@ class OutputParam: description: str = "" kwargs_type: str = None metadata: dict[str, Any] = None + # private, set by a composite block on its `intermediate_outputs` when the output is not always produced, because it + # comes from a conditional block that can be skipped, or that not every branch produces. Not for a leaf block to declare + _maybe: bool = field(default=False, compare=False) def __repr__(self): return ( diff --git a/tests/modular_pipelines/test_conditional_pipeline_blocks.py b/tests/modular_pipelines/test_conditional_pipeline_blocks.py index 34f3e60e6948..fe207620cbed 100644 --- a/tests/modular_pipelines/test_conditional_pipeline_blocks.py +++ b/tests/modular_pipelines/test_conditional_pipeline_blocks.py @@ -499,3 +499,60 @@ def test_branch_defaults_do_not_affect_sequential_blocks(self): state = pipe(prompt="p", image="i", strength=0.7) assert state.get("strength_seen_first") == 0.7 assert state.get("strength") == 0.7 + + +class ProducesLatentsAndImageLatentsBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def intermediate_outputs(self): + return [OutputParam("latents"), OutputParam("image_latents")] + + def __call__(self, components, state): + return components, state + + +class ProducesLatentsBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("image", required=True)] + + @property + def intermediate_outputs(self): + return [OutputParam("latents")] + + def __call__(self, components, state): + return components, state + + +class TestConditionalBlocksOptionalOutputs: + def _blocks(self, default_block_name="text2img"): + class Blocks(ConditionalPipelineBlocks): + block_classes = [ProducesLatentsBlock, ProducesLatentsAndImageLatentsBlock] + block_names = ["img2img", "text2img"] + block_trigger_inputs = ["image"] + + def select_block(self, image=None): + return "img2img" if image is not None else None + + Blocks.default_block_name = default_block_name + return Blocks() + + def test_output_is_always_produced_only_when_every_branch_produces_it(self): + blocks = self._blocks() + # `latents` comes out of both branches, `image_latents` out of `text2img` only + assert {out.name: out._maybe for out in blocks.intermediate_outputs} == { + "latents": False, + "image_latents": True, + } + # the sub-blocks' own declarations are untouched + assert all(not out._maybe for out in ProducesLatentsAndImageLatentsBlock().intermediate_outputs) + + def test_every_output_of_a_skippable_block_is_a_maybe(self): + blocks = self._blocks(default_block_name=None) + assert {out.name: out._maybe for out in blocks.intermediate_outputs} == { + "latents": True, + "image_latents": True, + } diff --git a/tests/modular_pipelines/test_sequential_pipeline_blocks.py b/tests/modular_pipelines/test_sequential_pipeline_blocks.py index 1349db3c6966..b0f3b76979b3 100644 --- a/tests/modular_pipelines/test_sequential_pipeline_blocks.py +++ b/tests/modular_pipelines/test_sequential_pipeline_blocks.py @@ -111,6 +111,17 @@ def select_block(self, image=None): return "encode" if image is not None else None +class NeedsImageLatentsBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("image_latents", required=True)] + + def __call__(self, components, state): + return components, state + + def strength_auto_blocks(img2img_default, inpaint_default): """ An auto block whose two branches declare `strength` with their own default: `None` for no default, `REQUIRED` for @@ -357,6 +368,50 @@ def __call__(self, components, state): assert required_flags(blocks, "image", "image_latents") == {"image": False, "image_latents": False} assert blocks.required_inputs == [] + def test_skippable_block_nested_in_a_sequence_is_still_optional(self): + class Encoders(SequentialPipelineBlocks): + block_classes = [PrepareLatentsBlock, SkippableEncodeBlocks] + block_names = ["prepare_latents", "encode"] + + # the sequence reports what `encode` reports: `image_latents` is only sometimes produced + assert [(out.name, out._maybe) for out in Encoders().intermediate_outputs] == [ + ("latents", False), + ("image_latents", True), + ] + + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"encoders": Encoders(), "consume": NeedsImageLatentsBlock()} + ) + # so wrapping `encode` in a sequence does not hide it: `image_latents` stays listed, optional. `latents` is + # produced on every run and is not listed + assert [inp.name for inp in blocks.inputs] == ["latents", "batch_size", "image", "image_latents"] + assert required_flags(blocks, "image", "image_latents") == {"image": False, "image_latents": False} + + def test_branch_with_a_skippable_block_inside_does_not_always_produce(self): + class EncodersWithOptionalImage(SequentialPipelineBlocks): + block_classes = [PrepareLatentsBlock, SkippableEncodeBlocks] + block_names = ["prepare_latents", "encode"] + + class EncodersOrEncode(ConditionalPipelineBlocks): + block_classes = [EncodersWithOptionalImage, EncodeImageBlock] + block_names = ["optional", "always"] + block_trigger_inputs = ["mask"] + default_block_name = "optional" + + def select_block(self, mask=None): + return "always" if mask is not None else "optional" + + # `always` produces `image_latents` every time, `optional` only when `image` is passed, so the block as a whole + # only sometimes does; `latents` comes out of one branch only + assert {out.name: out._maybe for out in EncodersOrEncode().intermediate_outputs} == { + "latents": True, + "image_latents": True, + } + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"encode": EncodersOrEncode(), "consume": NeedsImageLatentsBlock()} + ) + assert required_flags(blocks, "image_latents") == {"image_latents": False} + def test_record_of_a_conditional_sub_block_is_prefixed_with_its_name(self): # `denoise` is a conditional block whose img2img branch requires `image_latents`, recorded as # {"img2img": REQUIRED}; the sequence keeps the record, keyed by `denoise.img2img`, and the input optional: From d5e61508743462706b6e648e5ffd4a13865ba69a Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sat, 14 Nov 2026 02:25:47 +0100 Subject: [PATCH 12/33] [modular] `*_overrides` are lists of declarations A declaration carries its own name, so the `{name: declaration}` dict repeated it and the two could disagree. Co-authored-by: wangzihao <47708655+ZihaoW123@users.noreply.github.com> Co-Authored-By: Claude Fable 5.2 --- .ai/references/modular.md | 14 +-- .../sequential_pipeline_blocks.md | 8 +- .../modular_pipelines/helios/denoise.py | 12 +-- .../ltx2/modular_blocks_ltx25.py | 6 +- .../minimax_h3/modular_blocks_minimax_h3.py | 6 +- .../modular_pipelines/modular_pipeline.py | 92 ++++++++++--------- .../modular_pipeline_utils.py | 24 +++-- .../p_h3_image/modular_blocks_p_h3_image.py | 8 +- .../modular_blocks_stable_diffusion_xl.py | 2 +- .../wan/modular_blocks_wan22_i2v.py | 8 +- .../wan/modular_blocks_wan_i2v.py | 10 +- .../wan/modular_blocks_wan_vace.py | 8 +- .../test_sequential_pipeline_blocks.py | 40 +++++--- 13 files changed, 131 insertions(+), 107 deletions(-) diff --git a/.ai/references/modular.md b/.ai/references/modular.md index f1e240b35748..b0f9ccff0b49 100644 --- a/.ai/references/modular.md +++ b/.ai/references/modular.md @@ -300,14 +300,14 @@ The other two ways to declare an input: (1) `required=True` when the block cannot work without the value, so the caller has to pass it; the block raises `ValueError("Required input 'x' is missing")` when it is not passed or `None`. (2) Optional with no default when the block works with `None`. Most often that is the computed case above, and the block should then also declare the value as an output. -**An assembled block derives its declarations from its sub-blocks, and can redefine them with `*_overrides`.** Inputs, outputs, components and configs are declared on leaf blocks. A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its own from its sub-blocks. It cannot add or remove any of them, they are part of the blocks' logic, but it can redefine how one is declared: the default or description of an input, the repo and loading info of a component, the value of a config. You set these as class attributes on the composite, or pass them to `from_blocks_dict`: `input_overrides`, `component_overrides` and `config_overrides`, each a `{name: declaration}` dict. The name must be one the composite already derives, and the declaration takes the derived entry's place (gotcha #13). `intermediate_outputs` has no override: a composite can define the `outputs` property to choose the ones it lists and how it describes them. +**An assembled block derives its declarations from its sub-blocks, and can redefine them with `*_overrides`.** Inputs, outputs, components and configs are declared on leaf blocks. A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its own from its sub-blocks. It cannot add or remove any of them, they are part of the blocks' logic, but it can redefine how one is declared: the default or description of an input, the repo and loading info of a component, the value of a config. You set these as class attributes on the composite, or pass them to `from_blocks_dict`: `input_overrides`, `component_overrides` and `config_overrides`, each a list of declarations. The name of a declaration must be one the composite already derives, and the declaration takes the derived entry's place (gotcha #13). `intermediate_outputs` has no override: a composite can define the `outputs` property to choose the ones it lists and how it describes them. ```python class DistilledCoreDenoiseStep(SequentialPipelineBlocks): block_classes = [...] block_names = [...] # The distilled checkpoint samples in few steps. - input_overrides = {"num_inference_steps": InputParam.template("num_inference_steps", default=10)} + input_overrides = [InputParam.template("num_inference_steps", default=10)] ``` `component_overrides` and `config_overrides` work the same way with a `ComponentSpec` / `ConfigSpec`, see the ComponentSpec section below. Typical uses: set a preset's default as above, settle a default two sub-blocks disagree on, or reword a description from the composite's point of view. @@ -343,10 +343,10 @@ class DistilledBlocks(SequentialPipelineBlocks): block_classes = [TextEncoderStep, DenoiseStep, DecodeStep] block_names = ["text_encoder", "denoise", "decode"] # the distilled checkpoint has its own transformer weights and skips guidance - component_overrides = { - "transformer": ComponentSpec("transformer", YourTransformerModel, repo="org/model-distilled", subfolder="transformer"), - } - config_overrides = {"is_distilled": ConfigSpec("is_distilled", default=True)} + component_overrides = [ + ComponentSpec("transformer", YourTransformerModel, repo="org/model-distilled", subfolder="transformer"), + ] + config_overrides = [ConfigSpec("is_distilled", default=True)] ``` ## Gotchas @@ -378,7 +378,7 @@ class DistilledBlocks(SequentialPipelineBlocks): 12. **Helpers taking `components`.** Prefer explicit args, for example, `get_xx_prompt_embeds(text_encoder, tokenizer, prompt)` instead of `get_xx_prompt_embeds(components, prompt)`. Helpers and block methods should take only what they use so dependencies are visible and they can run without an assembled pipeline. Leave the full `components` object to `__call__`, which unpacks what its helpers need. See `get_qwen_prompt_embeds` in `qwenimage/encoders.py`. -13. **An override declaration that restates less than the derived entry.** `input_overrides = {"num_inference_steps": InputParam("num_inference_steps", default=10)}` replaces the derived entry whole, so the type hint and description the sub-block declared are gone. Build the override from the template, `InputParam.template("num_inference_steps", default=10)`, or restate everything. +13. **An override declaration that restates less than the derived entry.** `input_overrides = [InputParam("num_inference_steps", default=10)]` replaces the derived entry whole, so the type hint and description the sub-block declared are gone. Build the override from the template, `InputParam.template("num_inference_steps", default=10)`, or restate everything. ## Conversion checklist diff --git a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md index 39b1ffeb8b5a..b40418bd2430 100644 --- a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md +++ b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md @@ -110,7 +110,7 @@ class ImageProcessingStep(SequentialPipelineBlocks): " - Encodes the image into latent space." ) ``` -When you create a [`~modular_pipelines.SequentialPipelineBlocks`], `inputs`, `intermediate_outputs`, `expected_components` and `expected_configs` are aggregated from the sub-blocks, so there is no need to define them again. The sequence cannot add or remove any of them, but it can redefine how an input, a component or a config is declared with `input_overrides`, `component_overrides` and `config_overrides`. That lets you reuse the same sub-blocks with another default for an input, or other repo info for a component. Each is a `{name: declaration}` dict: the name must already be in the aggregated list, and the declaration takes the aggregated entry's place. +When you create a [`~modular_pipelines.SequentialPipelineBlocks`], `inputs`, `intermediate_outputs`, `expected_components` and `expected_configs` are aggregated from the sub-blocks, so there is no need to define them again. The sequence cannot add or remove any of them, but it can redefine how an input, a component or a config is declared with `input_overrides`, `component_overrides` and `config_overrides`. That lets you reuse the same sub-blocks with another default for an input, or other repo info for a component. Each is a list of declarations: the name of a declaration must already be in the aggregated list, and the declaration takes the aggregated entry's place. ```py class ImageProcessingStep(SequentialPipelineBlocks): @@ -121,11 +121,11 @@ class ImageProcessingStep(SequentialPipelineBlocks): block_classes = [InputBlock(), ImageEncoderBlock()] block_names = ["input", "image_encoder"] # `InputBlock` declares `num_images_per_prompt` without a default; this step defaults it to 1 - input_overrides = { - "num_images_per_prompt": InputParam( + input_overrides = [ + InputParam( name="num_images_per_prompt", type_hint=int, default=1, description="number of images per prompt" ) - } + ] ``` When two sub-blocks declare different defaults for the same input, the sequence keeps the first one and warns; an `input_overrides` entry states the default the whole sequence uses. When the input belongs to a conditional sub-block whose branches declare different defaults, the override gives the sequence one value for every workflow instead of the per-branch ones. diff --git a/src/diffusers/modular_pipelines/helios/denoise.py b/src/diffusers/modular_pipelines/helios/denoise.py index 8d28c2ac0934..553c85d45b47 100644 --- a/src/diffusers/modular_pipelines/helios/denoise.py +++ b/src/diffusers/modular_pipelines/helios/denoise.py @@ -1062,14 +1062,14 @@ class HeliosPyramidDistilledChunkDenoiseStep(HeliosChunkLoopWrapper): ] block_names = ["history_slice", "noise_gen", "denoise_inner", "update_chunk"] # the shared noise-gen step declares the non-distilled schedule; the distilled denoiser runs 2 steps per stage - input_overrides = { - "pyramid_num_inference_steps_list": InputParam( + input_overrides = [ + InputParam( "pyramid_num_inference_steps_list", default=[2, 2, 2], type_hint=list, description="Number of denoising steps per pyramid stage.", ) - } + ] @property def description(self) -> str: @@ -1090,14 +1090,14 @@ class HeliosPyramidDistilledI2VChunkDenoiseStep(HeliosChunkLoopWrapper): ] block_names = ["history_slice", "noise_gen", "denoise_inner", "update_chunk"] # the shared noise-gen step declares the non-distilled schedule; the distilled denoiser runs 2 steps per stage - input_overrides = { - "pyramid_num_inference_steps_list": InputParam( + input_overrides = [ + InputParam( "pyramid_num_inference_steps_list", default=[2, 2, 2], type_hint=list, description="Number of denoising steps per pyramid stage.", ) - } + ] @property def description(self) -> str: diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py index 2a3c40267812..e09405f033bc 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py @@ -369,8 +369,8 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): ] # `condition_encoder` declares `num_frames` without a default (the duration head predicts it) and `decode` # defaults it to 121, so the pipeline declares it itself: no default, auto-predicted when omitted - input_overrides = { - "num_frames": InputParam( + input_overrides = [ + InputParam( "num_frames", type_hint=int, default=None, @@ -379,7 +379,7 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): "(see `LTX2AutoDurationStep`)." ), ) - } + ] # `num_frames` on `in_context` is a requirement, not a trigger: the in-context checkpoints ship without a # `duration_head`, so the workflow drops `LTX2AutoDurationStep` and `LTX2ConditionEncoderStep` raises if # `num_frames` is still `None`. Matches `LTX2InContextBlocks`, which omits the duration step outright. diff --git a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py index db5bf991687a..14f2565b7496 100644 --- a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py +++ b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py @@ -759,8 +759,8 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): ] # `ref2va` requires `num_frames` (the references are truncated to it) while `t2va` and `fl2va` default it to 124, # so the pipeline declares it itself: optional, no default, each workflow keeps its own - input_overrides = { - "num_frames": InputParam( + input_overrides = [ + InputParam( name="num_frames", type_hint=int, description=( @@ -769,7 +769,7 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): "as long as a reference soundtrack, pass `round(samples / sample_rate * 24)`." ), ) - } + ] # One repository holds both checkpoint partitions (`transformer/` for `t2va`/`fl2va`, `transformer_ref/` for # `ref2va`), so the tasks are workflows over one shared `modular_model_index.json`, each declaring only the # components its own half loads. The keys below are the task names a `workflow=` argument to diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index a782e42bbcd3..3f2c15218f72 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -700,26 +700,26 @@ class ConditionalPipelineBlocks(ModularPipelineBlocks): default_block_name: Name of the default block to run when no trigger inputs match. If None, this block can be skipped entirely when no trigger inputs are provided. input_overrides: the composite block's own declarations for inputs it derives from its sub-blocks, as - `{name: InputParam}`, e.g. `{"strength": InputParam.template("strength", default=0.9999)}`. The name must - be a derived input, and the declaration replaces the derived one whole, so it states everything: default, + a list of `InputParam`, e.g. `[InputParam.template("strength", default=0.9999)]`. Each name must be a + derived input, and the declaration replaces the derived one whole, so it states everything: default, `required`, description. When the sub-blocks declare different defaults for the same input, each one keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with one value. component_overrides: the composite block's own specs for components it derives from its sub-blocks, as - `{name: ComponentSpec}`, e.g. `{"text_encoder": ComponentSpec("text_encoder", T5EncoderModel)}`. The name - must be a derived component. + a list of `ComponentSpec`, e.g. `[ComponentSpec("text_encoder", T5EncoderModel)]`. Each name must be a + derived component. config_overrides: the composite block's own specs for configs it derives from its sub-blocks, as - `{name: ConfigSpec}`, e.g. `{"vae_scale_factor": ConfigSpec("vae_scale_factor", default=16)}`. The name - must be a derived config. + a list of `ConfigSpec`, e.g. `[ConfigSpec("vae_scale_factor", default=16)]`. Each name must be a derived + config. """ block_classes = [] block_names = [] block_trigger_inputs = [] default_block_name = None - input_overrides: dict[str, InputParam] = {} - component_overrides: dict[str, ComponentSpec] = {} - config_overrides: dict[str, ConfigSpec] = {} + input_overrides: list[InputParam] = [] + component_overrides: list[ComponentSpec] = [] + config_overrides: list[ConfigSpec] = [] def __init__(self): sub_blocks = InsertableDict() @@ -1133,24 +1133,24 @@ class SequentialPipelineBlocks(ModularPipelineBlocks): block_classes: list of block classes to be used block_names: list of prefixes for each block input_overrides: the composite block's own declarations for inputs it derives from its sub-blocks, as - `{name: InputParam}`, e.g. `{"strength": InputParam.template("strength", default=0.9999)}`. The name must - be a derived input, and the declaration replaces the derived one whole, so it states everything: default, + a list of `InputParam`, e.g. `[InputParam.template("strength", default=0.9999)]`. Each name must be a + derived input, and the declaration replaces the derived one whole, so it states everything: default, `required`, description. When the sub-blocks declare different defaults for the same input, each one keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with one value. Can also be passed to `from_blocks_dict`. component_overrides: the composite block's own specs for components it derives from its sub-blocks, as - `{name: ComponentSpec}`, e.g. `{"text_encoder": ComponentSpec("text_encoder", T5EncoderModel)}`. The name - must be a derived component. Can also be passed to `from_blocks_dict`. + a list of `ComponentSpec`, e.g. `[ComponentSpec("text_encoder", T5EncoderModel)]`. Each name must be a + derived component. Can also be passed to `from_blocks_dict`. config_overrides: the composite block's own specs for configs it derives from its sub-blocks, as - `{name: ConfigSpec}`, e.g. `{"vae_scale_factor": ConfigSpec("vae_scale_factor", default=16)}`. The name - must be a derived config. Can also be passed to `from_blocks_dict`. + a list of `ConfigSpec`, e.g. `[ConfigSpec("vae_scale_factor", default=16)]`. Each name must be a derived + config. Can also be passed to `from_blocks_dict`. """ block_classes = [] block_names = [] - input_overrides: dict[str, InputParam] = {} - component_overrides: dict[str, ComponentSpec] = {} - config_overrides: dict[str, ConfigSpec] = {} + input_overrides: list[InputParam] = [] + component_overrides: list[ComponentSpec] = [] + config_overrides: list[ConfigSpec] = [] @property def description(self): @@ -1215,19 +1215,19 @@ def from_blocks_dict( cls, blocks_dict: dict[str, Any], description: str | None = None, - input_overrides: dict[str, InputParam] | None = None, - component_overrides: dict[str, ComponentSpec] | None = None, - config_overrides: dict[str, ConfigSpec] | None = None, + input_overrides: list[InputParam] | None = None, + component_overrides: list[ComponentSpec] | None = None, + config_overrides: list[ConfigSpec] | None = None, ) -> "SequentialPipelineBlocks": """Creates a SequentialPipelineBlocks instance from a dictionary of blocks. Args: blocks_dict: Dictionary mapping block names to block classes or instances description: Description of the new block - input_overrides: the composite's own declarations replacing derived `inputs`, as `{name: InputParam}` - component_overrides: the composite's own specs replacing derived `expected_components`, as - `{name: ComponentSpec}` - config_overrides: the composite's own specs replacing derived `expected_configs`, as `{name: ConfigSpec}` + input_overrides: the composite's own declarations replacing derived `inputs`, a list of `InputParam` + component_overrides: the composite's own specs replacing derived `expected_components`, a list of + `ComponentSpec` + config_overrides: the composite's own specs replacing derived `expected_configs`, a list of `ConfigSpec` Returns: A new SequentialPipelineBlocks instance @@ -1284,6 +1284,9 @@ def _get_inputs(self): # maybe produced by an earlier block: a ConditionalPipelineBlocks that can be skipped, or whose branches do not # all produce the output, at any depth maybe_outputs = set() + # names declared in `input_overrides`. `getattr`: `override_attribute` below reports an entry that is not an + # `InputParam` + overridden = {getattr(inp, "name", None) for inp in self.input_overrides} for block_name, block in self.sub_blocks.items(): for inp in block.inputs: # kwargs inputs have no name, key them by "*_" @@ -1327,7 +1330,7 @@ def _get_inputs(self): ) if key not in maybe_outputs and not has_default: existing.required = True - elif not inp.required and not existing.required and key not in self.input_overrides: + elif not inp.required and not existing.required and key not in overridden: # if two blocks both declare the same input as optional, with different non-None defaults, # use the first one, but warn and ask user to set the `input_overrides` on the composite to explicitly set the default. # e.g. `strength` is 0.3 on `set_timesteps` and 0.9999 on `prepare_latents`: the pipeline uses @@ -1566,25 +1569,25 @@ class LoopSequentialPipelineBlocks(ModularPipelineBlocks): block_classes: list of block classes to be used block_names: list of prefixes for each block input_overrides: the composite block's own declarations for inputs it derives from its sub-blocks, as - `{name: InputParam}`, e.g. `{"strength": InputParam.template("strength", default=0.9999)}`. The name must - be a derived input, and the declaration replaces the derived one whole, so it states everything: default, + a list of `InputParam`, e.g. `[InputParam.template("strength", default=0.9999)]`. Each name must be a + derived input, and the declaration replaces the derived one whole, so it states everything: default, `required`, description. When the sub-blocks declare different defaults for the same input, each one keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with one value. Can also be passed to `from_blocks_dict`. component_overrides: the composite block's own specs for components it derives from its sub-blocks, as - `{name: ComponentSpec}`, e.g. `{"text_encoder": ComponentSpec("text_encoder", T5EncoderModel)}`. The name - must be a derived component. Can also be passed to `from_blocks_dict`. + a list of `ComponentSpec`, e.g. `[ComponentSpec("text_encoder", T5EncoderModel)]`. Each name must be a + derived component. Can also be passed to `from_blocks_dict`. config_overrides: the composite block's own specs for configs it derives from its sub-blocks, as - `{name: ConfigSpec}`, e.g. `{"vae_scale_factor": ConfigSpec("vae_scale_factor", default=16)}`. The name - must be a derived config. Can also be passed to `from_blocks_dict`. + a list of `ConfigSpec`, e.g. `[ConfigSpec("vae_scale_factor", default=16)]`. Each name must be a derived + config. Can also be passed to `from_blocks_dict`. """ model_name = None block_classes = [] block_names = [] - input_overrides: dict[str, InputParam] = {} - component_overrides: dict[str, ComponentSpec] = {} - config_overrides: dict[str, ConfigSpec] = {} + input_overrides: list[InputParam] = [] + component_overrides: list[ComponentSpec] = [] + config_overrides: list[ConfigSpec] = [] @property def description(self) -> str: @@ -1642,6 +1645,9 @@ def _get_inputs(self): # blocks, so there is no skippable ConditionalPipelineBlocks whose outputs may or may not exist. inputs = {} outputs = set() # produced by an earlier block + # names declared in `input_overrides`. `getattr`: `override_attribute` below reports an entry that is not an + # `InputParam` + overridden = {getattr(inp, "name", None) for inp in self.input_overrides} for inp in self.loop_inputs: # kwargs inputs have no name, key them by "*_" like `ConditionalPipelineBlocks` does key = inp.name if inp.name is not None else f"*_{inp.kwargs_type}" @@ -1663,7 +1669,7 @@ def _get_inputs(self): not inputs[key].required and inp.default is not None and inp.default != inputs[key].default - and key not in self.input_overrides + and key not in overridden ): # two optional declarations: the earlier one wins, a later block that declares an actual default # which is not the one in use gets a warning (a later `None` takes whatever is set). Loop @@ -1731,19 +1737,19 @@ def __init__(self): def from_blocks_dict( cls, blocks_dict: dict[str, Any], - input_overrides: dict[str, InputParam] | None = None, - component_overrides: dict[str, ComponentSpec] | None = None, - config_overrides: dict[str, ConfigSpec] | None = None, + input_overrides: list[InputParam] | None = None, + component_overrides: list[ComponentSpec] | None = None, + config_overrides: list[ConfigSpec] | None = None, ) -> "LoopSequentialPipelineBlocks": """ Creates a LoopSequentialPipelineBlocks instance from a dictionary of blocks. Args: blocks_dict: Dictionary mapping block names to block instances - input_overrides: the composite's own declarations replacing derived `inputs`, as `{name: InputParam}` - component_overrides: the composite's own specs replacing derived `expected_components`, as - `{name: ComponentSpec}` - config_overrides: the composite's own specs replacing derived `expected_configs`, as `{name: ConfigSpec}` + input_overrides: the composite's own declarations replacing derived `inputs`, a list of `InputParam` + component_overrides: the composite's own specs replacing derived `expected_components`, a list of + `ComponentSpec` + config_overrides: the composite's own specs replacing derived `expected_configs`, a list of `ConfigSpec` Returns: A new LoopSequentialPipelineBlocks instance diff --git a/src/diffusers/modular_pipelines/modular_pipeline_utils.py b/src/diffusers/modular_pipelines/modular_pipeline_utils.py index c09db0f9bd1b..3365c6c74ada 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline_utils.py +++ b/src/diffusers/modular_pipelines/modular_pipeline_utils.py @@ -1129,14 +1129,14 @@ def combine_outputs(*named_output_lists: list[tuple[str, list[OutputParam]]]) -> return list(combined_dict.values()) -def override_attribute(items: list, overrides: dict[str, Any], attr_name: str, block_name: str) -> list: +def override_attribute(items: list, overrides: list, attr_name: str, block_name: str) -> list: """ Apply a composite block's `*_overrides` to the list it derived from its sub-blocks. Args: items: the derived `InputParam`, `ComponentSpec` or `ConfigSpec` list - overrides: `{name: declaration}`; every name must be in `items`, and the declaration, an object of the same - class as the entry it replaces, takes that entry's place whole + overrides: a list of declarations; the name of each must be in `items`, and the declaration, an object of the + same class as the entry it replaces, takes that entry's place whole attr_name: the overrides attribute being applied, e.g. `"input_overrides"`, for error messages block_name: the composite's class name, for error messages @@ -1148,7 +1148,7 @@ class as the entry it replaces, takes that entry's place whole ```python inputs = [InputParam("strength", default=0.3), InputParam("prompt", required=True)] override_attribute( - inputs, {"strength": InputParam.template("strength", default=0.9999)}, "input_overrides", "InpaintStep" + inputs, [InputParam.template("strength", default=0.9999)], "input_overrides", "InpaintStep" ) # [, ] ``` @@ -1160,7 +1160,17 @@ class as the entry it replaces, takes that entry's place whole item.name if item.name is not None else f"*_{item.kwargs_type}": position for position, item in enumerate(items) } - for name, declaration in overrides.items(): + replaced = set() + for declaration in overrides: + if not isinstance(declaration, (InputParam, ComponentSpec, ConfigSpec)): + raise TypeError( + f"{block_name}.{attr_name} must be a list of declarations, e.g. " + f'`[InputParam("strength", default=0.5)]`; got an entry of type {type(declaration).__name__}.' + ) + name = declaration.name if declaration.name is not None else f"*_{declaration.kwargs_type}" + if name in replaced: + raise ValueError(f"{block_name}.{attr_name} declares '{name}' more than once.") + replaced.add(name) if name not in index: raise ValueError( f"{block_name}.{attr_name} has an entry for '{name}', which this block does not derive from its " @@ -1169,8 +1179,8 @@ class as the entry it replaces, takes that entry's place whole expected = type(items[index[name]]) if not isinstance(declaration, expected): raise TypeError( - f"{block_name}.{attr_name}['{name}'] must be of type {expected.__name__}, replacing the derived one; " - f"got {type(declaration).__name__}." + f"The entry for '{name}' in {block_name}.{attr_name} must be of type {expected.__name__}, replacing " + f"the derived one; got {type(declaration).__name__}." ) items[index[name]] = declaration return items diff --git a/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py b/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py index 8baad8a3983e..7f3926d8ab4d 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py +++ b/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py @@ -453,8 +453,8 @@ class PH3ImageBlocks(SequentialPipelineBlocks): # `before_encode` reads `height`/`width` with no default (an edit derives them from the image) and the prompt # upsampler defaults them to 2912, so the pipeline declares them itself: optional, no default, each workflow keeps # its own - input_overrides = { - "height": InputParam.template( + input_overrides = [ + InputParam.template( "height", description=( "Height of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for " @@ -462,7 +462,7 @@ class PH3ImageBlocks(SequentialPipelineBlocks): "least `canvas_min_pixels`, and works best at 4 or 8 megapixels." ), ), - "width": InputParam.template( + InputParam.template( "width", description=( "Width of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for " @@ -470,7 +470,7 @@ class PH3ImageBlocks(SequentialPipelineBlocks): "least `canvas_min_pixels`, and works best at 4 or 8 megapixels." ), ), - } + ] _workflow_map = { "text2image": {"prompt": True}, "image_conditioned": {"prompt": True, "image": True}, diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py index 45f0acfbff97..e7016a0d5ed0 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py @@ -128,7 +128,7 @@ class StableDiffusionXLInpaintBeforeDenoiseStep(SequentialPipelineBlocks): ] block_names = ["set_timesteps", "prepare_latents", "prepare_add_cond"] # set_timesteps is shared with img2img and declares 0.3; inpainting starts from (almost) pure noise - input_overrides = {"strength": InputParam.template("strength", default=0.9999)} + input_overrides = [InputParam.template("strength", default=0.9999)] @property def description(self): diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py index 6f92a3085131..e3bc4583c90e 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py @@ -236,11 +236,9 @@ class Wan22Image2VideoBlocks(SequentialPipelineBlocks): ] # `vae_encoder` defaults `num_frames` to 81 and the denoise step declares it without a default (it fills it in # from the latents or the components), so the pipeline declares it itself - input_overrides = { - "num_frames": InputParam( - "num_frames", type_hint=int, default=81, description="The number of frames in the generated video." - ) - } + input_overrides = [ + InputParam("num_frames", type_hint=int, default=81, description="The number of frames in the generated video.") + ] @property def description(self): diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py index ac834eb37d48..6e8308bf5bdc 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py @@ -502,13 +502,13 @@ class WanImage2VideoAutoBlocks(SequentialPipelineBlocks): # aspect ratio) and `num_frames` to 81, while the denoise step declares them without a default (it fills them in # from the latents or the components). Both encoders are auto blocks the walk cannot count on, so the pipeline # declares the three itself - input_overrides = { - "height": InputParam("height", type_hint=int, default=480, description="The target height of the video."), - "width": InputParam("width", type_hint=int, default=832, description="The target width of the video."), - "num_frames": InputParam( + input_overrides = [ + InputParam("height", type_hint=int, default=480, description="The target height of the video."), + InputParam("width", type_hint=int, default=832, description="The target width of the video."), + InputParam( "num_frames", type_hint=int, default=81, description="The number of frames in the generated video." ), - } + ] _workflow_map = { "image2video": {"image": True, "prompt": True}, diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py index f3bd2ee4ce25..d54cfd8f0186 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py @@ -198,11 +198,9 @@ class Wan22VaceBlocks(SequentialPipelineBlocks): ] # `vace_encoder` defaults `num_frames` to 81 and the denoise step declares it without a default (it fills it in # from the latents or the components), so the pipeline declares it itself - input_overrides = { - "num_frames": InputParam( - "num_frames", type_hint=int, default=81, description="The number of frames in the generated video." - ) - } + input_overrides = [ + InputParam("num_frames", type_hint=int, default=81, description="The number of frames in the generated video.") + ] @property def description(self): diff --git a/tests/modular_pipelines/test_sequential_pipeline_blocks.py b/tests/modular_pipelines/test_sequential_pipeline_blocks.py index b0f3b76979b3..718709cbcd92 100644 --- a/tests/modular_pipelines/test_sequential_pipeline_blocks.py +++ b/tests/modular_pipelines/test_sequential_pipeline_blocks.py @@ -655,7 +655,7 @@ def test_default_override_settles_the_conflict(self): class Blocks(SequentialPipelineBlocks): block_classes = [FirstBlock, SecondBlock] block_names = ["first", "second"] - input_overrides = {"strength": InputParam("strength", default=0.9999)} + input_overrides = [InputParam("strength", default=0.9999)] with quiet(): inputs = Blocks().inputs @@ -696,7 +696,7 @@ def loop_inputs(self): with quiet(): inputs = Loop.from_blocks_dict( {"first": FirstBlock(), "second": SecondBlock()}, - input_overrides={"strength": InputParam("strength", default=0.9999)}, + input_overrides=[InputParam("strength", default=0.9999)], ).inputs assert [inp.default for inp in inputs if inp.name == "strength"] == [0.9999] @@ -821,14 +821,14 @@ class Blocks(SequentialPipelineBlocks): return Blocks() def test_input_override_replaces_the_derived_declaration(self): - blocks = self._sequential(input_overrides={"batch_size": InputParam("batch_size", default=1)}) + blocks = self._sequential(input_overrides=[InputParam("batch_size", default=1)]) (batch_size,) = [inp for inp in blocks.inputs if inp.name == "batch_size"] assert batch_size.default == 1 assert batch_size.required is False assert "batch_size" not in blocks.required_inputs def test_override_does_not_mutate_the_sub_block_declaration(self): - blocks = self._sequential(input_overrides={"batch_size": InputParam("batch_size", default=1)}) + blocks = self._sequential(input_overrides=[InputParam("batch_size", default=1)]) _ = blocks.inputs (batch_size,) = [inp for inp in blocks.sub_blocks["prepare_latents"].inputs if inp.name == "batch_size"] assert batch_size.required is True @@ -836,37 +836,49 @@ def test_override_does_not_mutate_the_sub_block_declaration(self): def test_component_and_config_overrides(self): blocks = self._sequential( - component_overrides={"unet": ComponentSpec("unet", description="any UNet2DConditionModel")}, - config_overrides={"scale": ConfigSpec("scale", default=16)}, + component_overrides=[ComponentSpec("unet", description="any UNet2DConditionModel")], + config_overrides=[ConfigSpec("scale", default=16)], ) assert [c.description for c in blocks.expected_components if c.name == "unet"] == ["any UNet2DConditionModel"] assert [c.default for c in blocks.expected_configs if c.name == "scale"] == [16] def test_unknown_name_raises(self): - blocks = self._sequential(input_overrides={"strenght": InputParam("strenght", default=0.5)}) + blocks = self._sequential(input_overrides=[InputParam("strenght", default=0.5)]) with pytest.raises(ValueError, match="strenght"): blocks.inputs - blocks = self._sequential(component_overrides={"vae": ComponentSpec("vae", description="x")}) + blocks = self._sequential(component_overrides=[ComponentSpec("vae", description="x")]) with pytest.raises(ValueError, match="vae"): blocks.expected_components - def test_partial_dict_override_raises(self): - blocks = self._sequential(input_overrides={"batch_size": {"default": 1}}) + def test_entry_that_is_not_a_declaration_raises(self): + # a partial dict, or the `{name: declaration}` shape, is not a list of declarations + for overrides in ([{"default": 1}], {"batch_size": InputParam("batch_size", default=1)}): + blocks = self._sequential(input_overrides=overrides) + with pytest.raises(TypeError, match="must be a list of declarations"): + blocks.inputs + blocks = self._sequential(input_overrides=[ConfigSpec("batch_size", default=1)]) with pytest.raises(TypeError, match="must be of type InputParam"): blocks.inputs + def test_name_declared_twice_raises(self): + blocks = self._sequential( + input_overrides=[InputParam("batch_size", default=1), InputParam("batch_size", default=2)] + ) + with pytest.raises(ValueError, match="more than once"): + blocks.inputs + def test_from_blocks_dict_accepts_overrides(self): blocks = SequentialPipelineBlocks.from_blocks_dict( {"prepare_latents": PrepareLatentsBlock(), "unet": UNetBlock()}, - input_overrides={"batch_size": InputParam("batch_size", default=2)}, - config_overrides={"scale": ConfigSpec("scale", default=16)}, + input_overrides=[InputParam("batch_size", default=2)], + config_overrides=[ConfigSpec("scale", default=16)], ) assert [inp.default for inp in blocks.inputs if inp.name == "batch_size"] == [2] assert [c.default for c in blocks.expected_configs if c.name == "scale"] == [16] loop = LoopSequentialPipelineBlocks.from_blocks_dict( {"unet": UNetBlock()}, - component_overrides={"unet": ComponentSpec("unet", description="any UNet2DConditionModel")}, + component_overrides=[ComponentSpec("unet", description="any UNet2DConditionModel")], ) assert [c.description for c in loop.expected_components if c.name == "unet"] == ["any UNet2DConditionModel"] @@ -878,7 +890,7 @@ def test_auto_blocks_override_replaces_the_per_block_defaults(self): assert strength.defaults_by_block == {"inpaint": 0.9999, "img2img": 0.3} class AutoWithDefault(Auto): - input_overrides = {"strength": InputParam("strength", default=0.5)} + input_overrides = [InputParam("strength", default=0.5)] (strength,) = [inp for inp in AutoWithDefault().inputs if inp.name == "strength"] assert strength.default == 0.5 From d3408bf7dca8fcc42a2ab90c1d7f18b178f99e97 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sun, 15 Nov 2026 02:54:49 +0100 Subject: [PATCH 13/33] [modular] replace composite `*_overrides` with `ModularPipelineBlocks.new()` A composite's override only changed what the composite declared; the leaf that reads the input kept filling in its own default, so under an auto block the run did not match the declaration. `block.new(...)` returns a copy of a leaf block with some of its declarations replaced, and composites derive from that. Two sub-blocks that declare different defaults are no longer a warning: the first wins and the other is logged at debug level, the docstring shows what runs. Co-Authored-By: Claude Fable 5.1 --- .ai/references/modular.md | 37 ++- .../sequential_pipeline_blocks.md | 22 +- .../modular_pipelines/helios/denoise.py | 42 +-- .../ltx2/modular_blocks_ltx25.py | 15 +- .../minimax_h3/modular_blocks_minimax_h3.py | 17 +- .../modular_pipelines/modular_pipeline.py | 280 ++++++++++-------- .../modular_pipeline_utils.py | 57 ---- .../p_h3_image/modular_blocks_p_h3_image.py | 23 +- .../modular_blocks_stable_diffusion_xl.py | 6 +- .../modular_pipelines/wan/encoders.py | 16 +- .../wan/modular_blocks_wan22_i2v.py | 17 +- .../wan/modular_blocks_wan_i2v.py | 43 +-- .../wan/modular_blocks_wan_vace.py | 7 +- .../test_sequential_pipeline_blocks.py | 212 +++++++------ 14 files changed, 354 insertions(+), 440 deletions(-) diff --git a/.ai/references/modular.md b/.ai/references/modular.md index b0f9ccff0b49..e418bfb4fc33 100644 --- a/.ai/references/modular.md +++ b/.ai/references/modular.md @@ -300,22 +300,24 @@ The other two ways to declare an input: (1) `required=True` when the block cannot work without the value, so the caller has to pass it; the block raises `ValueError("Required input 'x' is missing")` when it is not passed or `None`. (2) Optional with no default when the block works with `None`. Most often that is the computed case above, and the block should then also declare the value as an output. -**An assembled block derives its declarations from its sub-blocks, and can redefine them with `*_overrides`.** Inputs, outputs, components and configs are declared on leaf blocks. A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its own from its sub-blocks. It cannot add or remove any of them, they are part of the blocks' logic, but it can redefine how one is declared: the default or description of an input, the repo and loading info of a component, the value of a config. You set these as class attributes on the composite, or pass them to `from_blocks_dict`: `input_overrides`, `component_overrides` and `config_overrides`, each a list of declarations. The name of a declaration must be one the composite already derives, and the declaration takes the derived entry's place (gotcha #13). `intermediate_outputs` has no override: a composite can define the `outputs` property to choose the ones it lists and how it describes them. +**An assembled block derives its declarations from its sub-blocks; to change one, change the sub-block with `new()`.** Inputs, outputs, components and configs are declared on leaf blocks. A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its own from its sub-blocks and cannot add, remove or redefine any of them. To reuse a leaf block where only what it declares differs (the default or description of an input, the repo and loading info of a component, the value of a config) create it with `new()`: `block.new(description=..., inputs=[...], intermediate_outputs=[...], outputs=[...], components=[...], configs=[...])` returns a copy of the block with the listed entries replaced by name. It cannot add or remove an entry, and it only works on a block without sub-blocks. The block's code is unchanged, so a block that reads other inputs or runs other code is a block of its own, not a `new()` of another (gotcha #13). `intermediate_outputs` of a composite has no override: a composite can define the `outputs` property to choose the ones it lists and how it describes them. ```python class DistilledCoreDenoiseStep(SequentialPipelineBlocks): - block_classes = [...] + block_classes = [ + # The distilled checkpoint samples in few steps. + SetTimestepsStep().new(inputs=[InputParam.template("num_inference_steps", default=10)]), + ..., + ] block_names = [...] - # The distilled checkpoint samples in few steps. - input_overrides = [InputParam.template("num_inference_steps", default=10)] ``` -`component_overrides` and `config_overrides` work the same way with a `ComponentSpec` / `ConfigSpec`, see the ComponentSpec section below. Typical uses: set a preset's default as above, settle a default two sub-blocks disagree on, or reword a description from the composite's point of view. +`components=` and `configs=` work the same way with a `ComponentSpec` / `ConfigSpec`, see the ComponentSpec section below. Typical uses: set a preset's default as above, settle a default two sub-blocks disagree on, or reword a description. How the composite derives a shared input, and when you need an override: -- In a conditional block, the sub-blocks may declare different defaults for the same input, and each is respected: e.g. for `strength`, the img2img branch defaults to 0.3 and the inpaint branch to 0.9999, the docstring of the conditional block says "depends on the workflow", and at runtime the branch that runs fills in its own. An `input_overrides` entry on the conditional block gives every branch the same default and cannot set one per branch. To change a single branch's default, override the input on that sub-block. -- In a sequential block, the first sub-block that declares a default sets it for every sub-block after it. When a later sub-block declares a different default, the sequence keeps the first one and warns that the default is ambiguous. State it explicitly: declare the input in `input_overrides` on the composite. +- In a conditional block, the sub-blocks may declare different defaults for the same input, and each is respected: e.g. for `strength`, the img2img branch defaults to 0.3 and the inpaint branch to 0.9999, the docstring of the conditional block says "depends on the workflow", and at runtime the branch that runs fills in its own. To give every branch one value, use `new()` on the branch blocks that declare it. +- In a sequential block, the first sub-block that declares a default sets it for every sub-block after it. When a later sub-block declares a different default, the sequence keeps the first one and logs the other at debug level; make the two sub-blocks agree, with `new()` on one of them if the shared block is right elsewhere. A later default after a sub-block that leaves the input open (no default) is not a conflict: the sequence keeps the input open. - The `required` flag the composite reports is its own view, whether the caller has to pass the value. In a sequential block, a sub-block may require an input that an earlier sub-block defaults or may produce; the composite then lists it as optional. In a conditional block, an input is required only when every branch requires it and the block always runs. The sub-blocks' own declarations are untouched and still checked at runtime in `get_block_state`. **`outputs` is what the assembled block presents as its product.** By default it is every intermediate output. Override the property to list only what the step is for, so the docstring shows the product instead of every internal intermediate (Wan-Animate-2's core denoise lists only `segment_frames`). It changes the docstring and `output_names` only: everything stays in the pipeline state, and `pipe(..., output=)` still fetches any intermediate, listed or not. @@ -336,17 +338,22 @@ ComponentSpec( ) ``` -A composite inherits its sub-blocks' `ComponentSpec`s and `ConfigSpec`s. To point one at another repo, loading method or config value for the assembled pipeline, redeclare it in `component_overrides` / `config_overrides` on the composite, the same way `input_overrides` redeclares an input (see InputParam / OutputParam above). +A composite inherits its sub-blocks' `ComponentSpec`s and `ConfigSpec`s. To point one at another repo, loading method or config value for the assembled pipeline, create the sub-block that declares it with `new(components=[...])` / `new(configs=[...])`, the same way `new(inputs=[...])` redeclares an input (see InputParam / OutputParam above). ```python class DistilledBlocks(SequentialPipelineBlocks): - block_classes = [TextEncoderStep, DenoiseStep, DecodeStep] - block_names = ["text_encoder", "denoise", "decode"] - # the distilled checkpoint has its own transformer weights and skips guidance - component_overrides = [ - ComponentSpec("transformer", YourTransformerModel, repo="org/model-distilled", subfolder="transformer"), + block_classes = [ + TextEncoderStep, + # the distilled checkpoint has its own transformer weights and skips guidance + DenoiseStep().new( + components=[ + ComponentSpec("transformer", YourTransformerModel, repo="org/model-distilled", subfolder="transformer") + ], + configs=[ConfigSpec("is_distilled", default=True)], + ), + DecodeStep, ] - config_overrides = [ConfigSpec("is_distilled", default=True)] + block_names = ["text_encoder", "denoise", "decode"] ``` ## Gotchas @@ -378,7 +385,7 @@ class DistilledBlocks(SequentialPipelineBlocks): 12. **Helpers taking `components`.** Prefer explicit args, for example, `get_xx_prompt_embeds(text_encoder, tokenizer, prompt)` instead of `get_xx_prompt_embeds(components, prompt)`. Helpers and block methods should take only what they use so dependencies are visible and they can run without an assembled pipeline. Leave the full `components` object to `__call__`, which unpacks what its helpers need. See `get_qwen_prompt_embeds` in `qwenimage/encoders.py`. -13. **An override declaration that restates less than the derived entry.** `input_overrides = [InputParam("num_inference_steps", default=10)]` replaces the derived entry whole, so the type hint and description the sub-block declared are gone. Build the override from the template, `InputParam.template("num_inference_steps", default=10)`, or restate everything. +13. **A `new()` entry that restates less than the block's own.** `block.new(inputs=[InputParam("num_inference_steps", default=10)])` replaces the entry whole, so the type hint and description the block declared are gone. Build it from the template, `InputParam.template("num_inference_steps", default=10)`, or restate everything. And `new()` is only for declarations: a block that reads other inputs or runs other code is written as its own class, not as a subclass or a `new()` of another. ## Conversion checklist diff --git a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md index b40418bd2430..a6dc22eb0771 100644 --- a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md +++ b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md @@ -110,7 +110,7 @@ class ImageProcessingStep(SequentialPipelineBlocks): " - Encodes the image into latent space." ) ``` -When you create a [`~modular_pipelines.SequentialPipelineBlocks`], `inputs`, `intermediate_outputs`, `expected_components` and `expected_configs` are aggregated from the sub-blocks, so there is no need to define them again. The sequence cannot add or remove any of them, but it can redefine how an input, a component or a config is declared with `input_overrides`, `component_overrides` and `config_overrides`. That lets you reuse the same sub-blocks with another default for an input, or other repo info for a component. Each is a list of declarations: the name of a declaration must already be in the aggregated list, and the declaration takes the aggregated entry's place. +When you create a [`~modular_pipelines.SequentialPipelineBlocks`], `inputs`, `intermediate_outputs`, `expected_components` and `expected_configs` are aggregated from the sub-blocks, so there is no need to define them again. The sequence cannot add, remove or redefine any of them: they belong to the sub-blocks. To reuse a block with another default for an input, another description, or other repo info for a component, create the sub-block with `new()`. It returns a copy of the block with the listed declarations replaced, matched by name; the block's code is unchanged. ```py class ImageProcessingStep(SequentialPipelineBlocks): @@ -118,17 +118,21 @@ class ImageProcessingStep(SequentialPipelineBlocks): # auto_docstring """ model_name = "my_model" - block_classes = [InputBlock(), ImageEncoderBlock()] - block_names = ["input", "image_encoder"] - # `InputBlock` declares `num_images_per_prompt` without a default; this step defaults it to 1 - input_overrides = [ - InputParam( - name="num_images_per_prompt", type_hint=int, default=1, description="number of images per prompt" - ) + block_classes = [ + # `InputBlock` declares `num_images_per_prompt` without a default; this step defaults it to 1 + InputBlock().new( + inputs=[ + InputParam( + name="num_images_per_prompt", type_hint=int, default=1, description="number of images per prompt" + ) + ] + ), + ImageEncoderBlock(), ] + block_names = ["input", "image_encoder"] ``` -When two sub-blocks declare different defaults for the same input, the sequence keeps the first one and warns; an `input_overrides` entry states the default the whole sequence uses. When the input belongs to a conditional sub-block whose branches declare different defaults, the override gives the sequence one value for every workflow instead of the per-branch ones. +When two sub-blocks declare different defaults for the same input, the sequence keeps the first one (the other is logged at debug level). Make them agree: `new()` on the block whose default is right elsewhere gives it the value this sequence needs. A later default after a sub-block that leaves the input open is not a conflict; the sequence keeps the input open. A few properties you set yourself: diff --git a/src/diffusers/modular_pipelines/helios/denoise.py b/src/diffusers/modular_pipelines/helios/denoise.py index 553c85d45b47..6e6b916f58ca 100644 --- a/src/diffusers/modular_pipelines/helios/denoise.py +++ b/src/diffusers/modular_pipelines/helios/denoise.py @@ -1056,20 +1056,21 @@ class HeliosPyramidDistilledChunkDenoiseStep(HeliosChunkLoopWrapper): block_classes = [ HeliosChunkHistorySliceStep, - HeliosPyramidChunkNoiseGenStep, + # the shared noise-gen step defaults to the non-distilled schedule; the distilled denoiser runs 2 steps per stage + HeliosPyramidChunkNoiseGenStep().new( + inputs=[ + InputParam( + "pyramid_num_inference_steps_list", + default=[2, 2, 2], + type_hint=list, + description="Number of denoising steps per pyramid stage.", + ) + ] + ), HeliosPyramidDistilledChunkDenoiseInner, HeliosChunkUpdateStep, ] block_names = ["history_slice", "noise_gen", "denoise_inner", "update_chunk"] - # the shared noise-gen step declares the non-distilled schedule; the distilled denoiser runs 2 steps per stage - input_overrides = [ - InputParam( - "pyramid_num_inference_steps_list", - default=[2, 2, 2], - type_hint=list, - description="Number of denoising steps per pyramid stage.", - ) - ] @property def description(self) -> str: @@ -1084,20 +1085,21 @@ class HeliosPyramidDistilledI2VChunkDenoiseStep(HeliosChunkLoopWrapper): block_classes = [ HeliosI2VChunkHistorySliceStep, - HeliosPyramidChunkNoiseGenStep, + # the shared noise-gen step defaults to the non-distilled schedule; the distilled denoiser runs 2 steps per stage + HeliosPyramidChunkNoiseGenStep().new( + inputs=[ + InputParam( + "pyramid_num_inference_steps_list", + default=[2, 2, 2], + type_hint=list, + description="Number of denoising steps per pyramid stage.", + ) + ] + ), HeliosPyramidDistilledChunkDenoiseInner, HeliosChunkUpdateStep, ] block_names = ["history_slice", "noise_gen", "denoise_inner", "update_chunk"] - # the shared noise-gen step declares the non-distilled schedule; the distilled denoiser runs 2 steps per stage - input_overrides = [ - InputParam( - "pyramid_num_inference_steps_list", - default=[2, 2, 2], - type_hint=list, - description="Number of denoising steps per pyramid stage.", - ) - ] @property def description(self) -> str: diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py index e09405f033bc..1219b9a8d937 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py @@ -15,7 +15,7 @@ import torch from ..modular_pipeline import AutoPipelineBlocks, SequentialPipelineBlocks -from ..modular_pipeline_utils import InputParam, OutputParam +from ..modular_pipeline_utils import OutputParam from .decoders import ( LTX2AudioDecoderStep, LTX2DiffusionVaeDecoderStep, @@ -367,19 +367,6 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): "denoise", "decode", ] - # `condition_encoder` declares `num_frames` without a default (the duration head predicts it) and `decode` - # defaults it to 121, so the pipeline declares it itself: no default, auto-predicted when omitted - input_overrides = [ - InputParam( - "num_frames", - type_hint=int, - default=None, - description=( - "The number of frames in the generated video. Omit to auto-predict via the `duration_head` " - "(see `LTX2AutoDurationStep`)." - ), - ) - ] # `num_frames` on `in_context` is a requirement, not a trigger: the in-context checkpoints ship without a # `duration_head`, so the workflow drops `LTX2AutoDurationStep` and `LTX2ConditionEncoderStep` raises if # `num_frames` is still `None`. Matches `LTX2InContextBlocks`, which omits the duration step outright. diff --git a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py index 14f2565b7496..34f43bce421b 100644 --- a/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py +++ b/src/diffusers/modular_pipelines/minimax_h3/modular_blocks_minimax_h3.py @@ -15,7 +15,7 @@ import torch from ..modular_pipeline import ConditionalPipelineBlocks, SequentialPipelineBlocks -from ..modular_pipeline_utils import InputParam, OutputParam +from ..modular_pipeline_utils import OutputParam from .before_denoise import ( MiniMaxH3FL2VAPrepareLatentsStep, MiniMaxH3NoKeyframeAnchorsStep, @@ -692,7 +692,7 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): Height of the generated video in pixels, a multiple of 32. width (`int`, *optional*): Width of the generated video in pixels, a multiple of 32. - num_frames (`int`, *optional*): + num_frames (`int`, *optional*, depends on the workflow): Number of frames to generate, at the fixed 24 fps. Snapped up to the next `17 * n + 5` the video VAE can decode; the resulting duration must stay between 5 and 15 seconds. To generate a video as long as a reference soundtrack, pass `round(samples / sample_rate * 24)`. @@ -757,19 +757,6 @@ class MiniMaxH3Blocks(SequentialPipelineBlocks): "denoise", "decode", ] - # `ref2va` requires `num_frames` (the references are truncated to it) while `t2va` and `fl2va` default it to 124, - # so the pipeline declares it itself: optional, no default, each workflow keeps its own - input_overrides = [ - InputParam( - name="num_frames", - type_hint=int, - description=( - "Number of frames to generate, at the fixed 24 fps. Snapped up to the next `17 * n + 5` the video " - "VAE can decode; the resulting duration must stay between 5 and 15 seconds. To generate a video " - "as long as a reference soundtrack, pass `round(samples / sample_rate * 24)`." - ), - ) - ] # One repository holds both checkpoint partitions (`transformer/` for `t2va`/`fl2va`, `transformer_ref/` for # `ref2va`), so the tasks are workflows over one shared `modular_model_index.json`, each declaring only the # components its own half loads. The keys below are the task names a `workflow=` argument to diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 3f2c15218f72..12b82d714969 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -66,7 +66,6 @@ format_workflow, generate_modular_model_card_content, make_doc_string, - override_attribute, ) @@ -433,6 +432,129 @@ def inputs(self) -> list[InputParam]: """list of input parameters. Must be implemented by subclasses.""" return [] + def new( + self, + description: str | None = None, + inputs: list[InputParam] | None = None, + intermediate_outputs: list[OutputParam] | None = None, + outputs: list[OutputParam] | None = None, + components: list[ComponentSpec] | None = None, + configs: list[ConfigSpec] | None = None, + ) -> "ModularPipelineBlocks": + """ + Creates a copy of this block with some of its declarations replaced, e.g. another default for an input, so the + same block can be reused where only what it declares differs. The block's code is not changed and the copy + keeps what this block was created with. An entry cannot be added or removed: a block that reads other inputs + or runs other code is a block of its own. + + Args: + description: the description of the new block, in place of this block's + inputs: `InputParam`s replacing the ones the block declares under the same name. Each takes the declared + entry's place whole, so it states everything: default, `required`, description. + intermediate_outputs: the same for `intermediate_outputs`, a list of `OutputParam` + outputs: the same for `outputs`, a list of `OutputParam` + components: the same for `expected_components`, a list of `ComponentSpec` + configs: the same for `expected_configs`, a list of `ConfigSpec` + + Returns: + A new block of the same class that declares the replaced entries. + + Example: + ```python + class InpaintBeforeDenoiseStep(SequentialPipelineBlocks): + block_classes = [ + # shared with img2img, which defaults `strength` to 0.3 + SetTimestepsStep().new(inputs=[InputParam.template("strength", default=0.9999)]), + InpaintPrepareLatentsStep, + ] + block_names = ["set_timesteps", "prepare_latents"] + ``` + """ + block_class = self.__class__ + # a block assembled from sub-blocks (sequential, conditional, loop) derives its declarations from them + if self.sub_blocks: + raise TypeError( + f"`{block_class.__name__}.new()`: {block_class.__name__} derives its declarations from its " + f"sub-blocks. Call `new()` on the sub-block that declares the entry." + ) + + # the declarations of the new block, replaced one list at a time + properties = {} + + if description is not None: + properties["description"] = property(lambda self: description) + + if inputs: + # e.g. [, ] + new_inputs = list(self.inputs) + # kwargs inputs have no name and are keyed by "*_", like `ConditionalPipelineBlocks` does + names = [inp.name if inp.name is not None else f"*_{inp.kwargs_type}" for inp in new_inputs] + for inp in inputs: + name = inp.name if inp.name is not None else f"*_{inp.kwargs_type}" + if name not in names: + raise ValueError( + f"`{block_class.__name__}.new()`: {block_class.__name__} does not declare an input '{name}'. " + f"Available: {sorted(n for n in names if not n.startswith('*_'))}" + ) + # e.g. -> [, ] + new_inputs[names.index(name)] = inp + properties["inputs"] = property(lambda self: list(new_inputs)) + + if intermediate_outputs: + new_intermediate_outputs = list(self.intermediate_outputs) + names = [out.name for out in new_intermediate_outputs] + for out in intermediate_outputs: + if out.name not in names: + raise ValueError( + f"`{block_class.__name__}.new()`: {block_class.__name__} does not declare an intermediate " + f"output '{out.name}'. Available: {sorted(names)}" + ) + new_intermediate_outputs[names.index(out.name)] = out + properties["intermediate_outputs"] = property(lambda self: list(new_intermediate_outputs)) + + if outputs: + new_outputs = list(self.outputs) + names = [out.name for out in new_outputs] + for out in outputs: + if out.name not in names: + raise ValueError( + f"`{block_class.__name__}.new()`: {block_class.__name__} does not list an output " + f"'{out.name}' in `outputs`. Available: {sorted(names)}" + ) + new_outputs[names.index(out.name)] = out + properties["outputs"] = property(lambda self: list(new_outputs)) + + if components: + new_components = list(self.expected_components) + names = [spec.name for spec in new_components] + for spec in components: + if spec.name not in names: + raise ValueError( + f"`{block_class.__name__}.new()`: {block_class.__name__} does not declare a component " + f"'{spec.name}'. Available: {sorted(names)}" + ) + new_components[names.index(spec.name)] = spec + properties["expected_components"] = property(lambda self: list(new_components)) + + if configs: + new_configs = list(self.expected_configs) + names = [spec.name for spec in new_configs] + for spec in configs: + if spec.name not in names: + raise ValueError( + f"`{block_class.__name__}.new()`: {block_class.__name__} does not declare a config " + f"'{spec.name}'. Available: {sorted(names)}" + ) + new_configs[names.index(spec.name)] = spec + properties["expected_configs"] = property(lambda self: list(new_configs)) + + # the new block is a copy of this one, as an instance of a subclass with the same name that returns the lists + # above: the same as subclassing the block by hand to change what it declares + new_class = type(block_class.__name__, (block_class,), {"__module__": block_class.__module__, **properties}) + block = deepcopy(self) + block.__class__ = new_class + return block + def _get_required_inputs(self): input_names = [] for input_param in self.inputs: @@ -699,27 +821,12 @@ class ConditionalPipelineBlocks(ModularPipelineBlocks): element specifies the trigger input for the corresponding block. default_block_name: Name of the default block to run when no trigger inputs match. If None, this block can be skipped entirely when no trigger inputs are provided. - input_overrides: the composite block's own declarations for inputs it derives from its sub-blocks, as - a list of `InputParam`, e.g. `[InputParam.template("strength", default=0.9999)]`. Each name must be a - derived input, and the declaration replaces the derived one whole, so it states everything: default, - `required`, description. When the sub-blocks declare different defaults for the same input, each one - keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with - one value. - component_overrides: the composite block's own specs for components it derives from its sub-blocks, as - a list of `ComponentSpec`, e.g. `[ComponentSpec("text_encoder", T5EncoderModel)]`. Each name must be a - derived component. - config_overrides: the composite block's own specs for configs it derives from its sub-blocks, as - a list of `ConfigSpec`, e.g. `[ConfigSpec("vae_scale_factor", default=16)]`. Each name must be a derived - config. """ block_classes = [] block_names = [] block_trigger_inputs = [] default_block_name = None - input_overrides: list[InputParam] = [] - component_overrides: list[ComponentSpec] = [] - config_overrides: list[ConfigSpec] = [] def __init__(self): sub_blocks = InsertableDict() @@ -753,9 +860,7 @@ def expected_components(self): for component in block.expected_components: if component not in expected_components: expected_components.append(component) - return override_attribute( - expected_components, self.component_overrides, "component_overrides", self.__class__.__name__ - ) + return expected_components @property def expected_configs(self): @@ -764,7 +869,7 @@ def expected_configs(self): for config in block.expected_configs: if config not in expected_configs: expected_configs.append(config) - return override_attribute(expected_configs, self.config_overrides, "config_overrides", self.__class__.__name__) + return expected_configs def _get_inputs(self): # Merge the sub-blocks' inputs into one. Each sub-block is a branch and only one of them runs, so an input declared by @@ -837,7 +942,7 @@ def _get_inputs(self): # a fresh copy so the sub-blocks' own params are never mutated inputs.append(replace(input_param, default=None, required=False, defaults_by_block=defaults_by_block)) - return override_attribute(inputs, self.input_overrides, "input_overrides", self.__class__.__name__) + return inputs @property def inputs(self) -> list[tuple[str, Any]]: @@ -1132,25 +1237,10 @@ class SequentialPipelineBlocks(ModularPipelineBlocks): Attributes: block_classes: list of block classes to be used block_names: list of prefixes for each block - input_overrides: the composite block's own declarations for inputs it derives from its sub-blocks, as - a list of `InputParam`, e.g. `[InputParam.template("strength", default=0.9999)]`. Each name must be a - derived input, and the declaration replaces the derived one whole, so it states everything: default, - `required`, description. When the sub-blocks declare different defaults for the same input, each one - keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with - one value. Can also be passed to `from_blocks_dict`. - component_overrides: the composite block's own specs for components it derives from its sub-blocks, as - a list of `ComponentSpec`, e.g. `[ComponentSpec("text_encoder", T5EncoderModel)]`. Each name must be a - derived component. Can also be passed to `from_blocks_dict`. - config_overrides: the composite block's own specs for configs it derives from its sub-blocks, as - a list of `ConfigSpec`, e.g. `[ConfigSpec("vae_scale_factor", default=16)]`. Each name must be a derived - config. Can also be passed to `from_blocks_dict`. """ block_classes = [] block_names = [] - input_overrides: list[InputParam] = [] - component_overrides: list[ComponentSpec] = [] - config_overrides: list[ConfigSpec] = [] @property def description(self): @@ -1167,9 +1257,7 @@ def expected_components(self): for component in block.expected_components: if component not in expected_components: expected_components.append(component) - return override_attribute( - expected_components, self.component_overrides, "component_overrides", self.__class__.__name__ - ) + return expected_components @property def expected_configs(self): @@ -1178,7 +1266,7 @@ def expected_configs(self): for config in block.expected_configs: if config not in expected_configs: expected_configs.append(config) - return override_attribute(expected_configs, self.config_overrides, "config_overrides", self.__class__.__name__) + return expected_configs @property def available_workflows(self): @@ -1215,19 +1303,12 @@ def from_blocks_dict( cls, blocks_dict: dict[str, Any], description: str | None = None, - input_overrides: list[InputParam] | None = None, - component_overrides: list[ComponentSpec] | None = None, - config_overrides: list[ConfigSpec] | None = None, ) -> "SequentialPipelineBlocks": """Creates a SequentialPipelineBlocks instance from a dictionary of blocks. Args: blocks_dict: Dictionary mapping block names to block classes or instances description: Description of the new block - input_overrides: the composite's own declarations replacing derived `inputs`, a list of `InputParam` - component_overrides: the composite's own specs replacing derived `expected_components`, a list of - `ComponentSpec` - config_overrides: the composite's own specs replacing derived `expected_configs`, a list of `ConfigSpec` Returns: A new SequentialPipelineBlocks instance @@ -1248,13 +1329,6 @@ def from_blocks_dict( if description is not None: instance.description = description - for name, value in ( - ("input_overrides", input_overrides), - ("component_overrides", component_overrides), - ("config_overrides", config_overrides), - ): - if value is not None: - setattr(instance, name, value) return instance @@ -1284,9 +1358,6 @@ def _get_inputs(self): # maybe produced by an earlier block: a ConditionalPipelineBlocks that can be skipped, or whose branches do not # all produce the output, at any depth maybe_outputs = set() - # names declared in `input_overrides`. `getattr`: `override_attribute` below reports an entry that is not an - # `InputParam` - overridden = {getattr(inp, "name", None) for inp in self.input_overrides} for block_name, block in self.sub_blocks.items(): for inp in block.inputs: # kwargs inputs have no name, key them by "*_" @@ -1330,29 +1401,33 @@ def _get_inputs(self): ) if key not in maybe_outputs and not has_default: existing.required = True - elif not inp.required and not existing.required and key not in overridden: - # if two blocks both declare the same input as optional, with different non-None defaults, - # use the first one, but warn and ask user to set the `input_overrides` on the composite to explicitly set the default. - # e.g. `strength` is 0.3 on `set_timesteps` and 0.9999 on `prepare_latents`: the pipeline uses - # 0.3, and `input_overrides` on the composite picks another value - if inp.default is not None and inp.default != existing.default: + elif ( + not inp.required + and not existing.required + and inp.default is not None + and inp.default != existing.default + ): + # the earlier declaration wins. Two values, e.g. `strength` is 0.3 on `set_timesteps` and 0.9999 + # on `prepare_latents`: the pipeline uses 0.3, `set_timesteps.new(inputs=[...])` picks another + # value. An earlier declaration with no default stays open, e.g. an earlier block predicts + # `num_frames` when it is omitted and `decode` defaults it to 121 for when it runs on its own + if existing.defaults_by_block: + existing_text = f"per-branch defaults {existing.defaults_by_block!r}" + elif existing.default is None: + existing_text = "no default" + else: existing_text = f"default {existing.default!r}" - if existing.defaults_by_block: - existing_text = f"per-branch defaults {existing.defaults_by_block!r}" - logger.warning( - f"{self.__class__.__name__}: sub-block '{block_name}' declares default {inp.default!r} for " - f"input '{key}', but an earlier sub-block declares {existing_text}. The sequence keeps the " - f"earlier declaration. Declare '{key}' in `input_overrides` on {self.__class__.__name__} " - f"if that is not intended." - ) + logger.debug( + f"{self.__class__.__name__}: sub-block '{block_name}' declares default {inp.default!r} for " + f"input '{key}', but an earlier sub-block declares {existing_text}. The sequence keeps the " + f"earlier declaration." + ) # an output the block always produces goes to `outputs`, one marked `_maybe` (see `OutputParam`) to # `maybe_outputs`, e.g. `image_encoder` may be skipped, so `image_latents` is only produced if it runs for out in block.intermediate_outputs: (maybe_outputs if out._maybe else outputs).add(out.name) - return override_attribute( - list(inputs.values()), self.input_overrides, "input_overrides", self.__class__.__name__ - ) + return list(inputs.values()) @property def inputs(self) -> list[tuple[str, Any]]: @@ -1568,26 +1643,11 @@ class LoopSequentialPipelineBlocks(ModularPipelineBlocks): Attributes: block_classes: list of block classes to be used block_names: list of prefixes for each block - input_overrides: the composite block's own declarations for inputs it derives from its sub-blocks, as - a list of `InputParam`, e.g. `[InputParam.template("strength", default=0.9999)]`. Each name must be a - derived input, and the declaration replaces the derived one whole, so it states everything: default, - `required`, description. When the sub-blocks declare different defaults for the same input, each one - keeps its own at runtime (`defaults_by_block`); a declaration here with a default replaces them all with - one value. Can also be passed to `from_blocks_dict`. - component_overrides: the composite block's own specs for components it derives from its sub-blocks, as - a list of `ComponentSpec`, e.g. `[ComponentSpec("text_encoder", T5EncoderModel)]`. Each name must be a - derived component. Can also be passed to `from_blocks_dict`. - config_overrides: the composite block's own specs for configs it derives from its sub-blocks, as - a list of `ConfigSpec`, e.g. `[ConfigSpec("vae_scale_factor", default=16)]`. Each name must be a derived - config. Can also be passed to `from_blocks_dict`. """ model_name = None block_classes = [] block_names = [] - input_overrides: list[InputParam] = [] - component_overrides: list[ComponentSpec] = [] - config_overrides: list[ConfigSpec] = [] @property def description(self) -> str: @@ -1623,9 +1683,7 @@ def expected_components(self): for component in self.loop_expected_components: if component not in expected_components: expected_components.append(component) - return override_attribute( - expected_components, self.component_overrides, "component_overrides", self.__class__.__name__ - ) + return expected_components # modified from SequentialPipelineBlocks to include loop_expected_configs @property @@ -1638,16 +1696,13 @@ def expected_configs(self): for config in self.loop_expected_configs: if config not in expected_configs: expected_configs.append(config) - return override_attribute(expected_configs, self.config_overrides, "config_overrides", self.__class__.__name__) + return expected_configs def _get_inputs(self): # Same walk as `SequentialPipelineBlocks._get_inputs`, starting from `loop_inputs`. Loop sub-blocks are leaf # blocks, so there is no skippable ConditionalPipelineBlocks whose outputs may or may not exist. inputs = {} outputs = set() # produced by an earlier block - # names declared in `input_overrides`. `getattr`: `override_attribute` below reports an entry that is not an - # `InputParam` - overridden = {getattr(inp, "name", None) for inp in self.input_overrides} for inp in self.loop_inputs: # kwargs inputs have no name, key them by "*_" like `ConditionalPipelineBlocks` does key = inp.name if inp.name is not None else f"*_{inp.kwargs_type}" @@ -1665,26 +1720,17 @@ def _get_inputs(self): if inputs[key].default is None: # nothing earlier defaults or produces it: the caller has to pass it inputs[key].required = True - elif ( - not inputs[key].required - and inp.default is not None - and inp.default != inputs[key].default - and key not in overridden - ): - # two optional declarations: the earlier one wins, a later block that declares an actual default - # which is not the one in use gets a warning (a later `None` takes whatever is set). Loop + elif not inputs[key].required and inp.default is not None and inp.default != inputs[key].default: + # two optional declarations: the earlier one wins (a later `None` takes whatever is set). Loop # sub-blocks are leaves, so there are no per-branch defaults to consider - logger.warning( + logger.debug( f"{self.__class__.__name__}: sub-block '{block_name}' declares default {inp.default!r} for " f"input '{key}', but an earlier declaration (a loop input or an earlier sub-block) has " - f"{inputs[key].default!r}. The loop keeps the earlier declaration. Declare '{key}' in " - f"`input_overrides` on {self.__class__.__name__} if that is not intended." + f"{inputs[key].default!r}. The loop keeps the earlier declaration." ) outputs.update(out.name for out in block.intermediate_outputs) - return override_attribute( - list(inputs.values()), self.input_overrides, "input_overrides", self.__class__.__name__ - ) + return list(inputs.values()) @property # Copied from diffusers.modular_pipelines.modular_pipeline.SequentialPipelineBlocks.inputs @@ -1737,19 +1783,12 @@ def __init__(self): def from_blocks_dict( cls, blocks_dict: dict[str, Any], - input_overrides: list[InputParam] | None = None, - component_overrides: list[ComponentSpec] | None = None, - config_overrides: list[ConfigSpec] | None = None, ) -> "LoopSequentialPipelineBlocks": """ Creates a LoopSequentialPipelineBlocks instance from a dictionary of blocks. Args: blocks_dict: Dictionary mapping block names to block instances - input_overrides: the composite's own declarations replacing derived `inputs`, a list of `InputParam` - component_overrides: the composite's own specs replacing derived `expected_components`, a list of - `ComponentSpec` - config_overrides: the composite's own specs replacing derived `expected_configs`, a list of `ConfigSpec` Returns: A new LoopSequentialPipelineBlocks instance @@ -1767,13 +1806,6 @@ def from_blocks_dict( instance.block_classes = [block.__class__ for block in blocks_dict.values()] instance.block_names = list(blocks_dict.keys()) instance.sub_blocks = blocks_dict - for name, value in ( - ("input_overrides", input_overrides), - ("component_overrides", component_overrides), - ("config_overrides", config_overrides), - ): - if value is not None: - setattr(instance, name, value) return instance def loop_step(self, components, state: PipelineState, **kwargs): diff --git a/src/diffusers/modular_pipelines/modular_pipeline_utils.py b/src/diffusers/modular_pipelines/modular_pipeline_utils.py index 3365c6c74ada..209959059ee9 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline_utils.py +++ b/src/diffusers/modular_pipelines/modular_pipeline_utils.py @@ -1129,63 +1129,6 @@ def combine_outputs(*named_output_lists: list[tuple[str, list[OutputParam]]]) -> return list(combined_dict.values()) -def override_attribute(items: list, overrides: list, attr_name: str, block_name: str) -> list: - """ - Apply a composite block's `*_overrides` to the list it derived from its sub-blocks. - - Args: - items: the derived `InputParam`, `ComponentSpec` or `ConfigSpec` list - overrides: a list of declarations; the name of each must be in `items`, and the declaration, an object of the - same class as the entry it replaces, takes that entry's place whole - attr_name: the overrides attribute being applied, e.g. `"input_overrides"`, for error messages - block_name: the composite's class name, for error messages - - Returns: - `items` with each overridden entry replaced by the composite's own declaration; the sub-blocks' declarations - are not touched. - - Example: - ```python - inputs = [InputParam("strength", default=0.3), InputParam("prompt", required=True)] - override_attribute( - inputs, [InputParam.template("strength", default=0.9999)], "input_overrides", "InpaintStep" - ) - # [, ] - ``` - """ - if not overrides: - return items - # kwargs inputs/outputs have no name and are keyed by "*_", like `ConditionalPipelineBlocks` does - index = { - item.name if item.name is not None else f"*_{item.kwargs_type}": position - for position, item in enumerate(items) - } - replaced = set() - for declaration in overrides: - if not isinstance(declaration, (InputParam, ComponentSpec, ConfigSpec)): - raise TypeError( - f"{block_name}.{attr_name} must be a list of declarations, e.g. " - f'`[InputParam("strength", default=0.5)]`; got an entry of type {type(declaration).__name__}.' - ) - name = declaration.name if declaration.name is not None else f"*_{declaration.kwargs_type}" - if name in replaced: - raise ValueError(f"{block_name}.{attr_name} declares '{name}' more than once.") - replaced.add(name) - if name not in index: - raise ValueError( - f"{block_name}.{attr_name} has an entry for '{name}', which this block does not derive from its " - f"sub-blocks. Available: {sorted(k for k in index if not k.startswith('*_'))}" - ) - expected = type(items[index[name]]) - if not isinstance(declaration, expected): - raise TypeError( - f"The entry for '{name}' in {block_name}.{attr_name} must be of type {expected.__name__}, replacing " - f"the derived one; got {type(declaration).__name__}." - ) - items[index[name]] = declaration - return items - - def generate_modular_model_card_content(blocks) -> dict[str, Any]: """ Generate model card content for a modular pipeline. diff --git a/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py b/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py index 7f3926d8ab4d..c0361f188c1d 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py +++ b/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py @@ -15,7 +15,7 @@ import torch from ..modular_pipeline import AutoPipelineBlocks, SequentialPipelineBlocks -from ..modular_pipeline_utils import InputParam, OutputParam +from ..modular_pipeline_utils import OutputParam from .before_denoise import ( PH3ImageEditPrepareLatentsStep, PH3ImageEditPrepareLayoutStep, @@ -450,27 +450,6 @@ class PH3ImageBlocks(SequentialPipelineBlocks): PH3ImageDecodeStep, ] block_names = ["before_encode", "prompt_upsampler", "text_encoder", "vae_encoder", "denoise", "decode"] - # `before_encode` reads `height`/`width` with no default (an edit derives them from the image) and the prompt - # upsampler defaults them to 2912, so the pipeline declares them itself: optional, no default, each workflow keeps - # its own - input_overrides = [ - InputParam.template( - "height", - description=( - "Height of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for " - "`text2image`, which works best at 8 megapixels. An edit defaults to the first image's size, at " - "least `canvas_min_pixels`, and works best at 4 or 8 megapixels." - ), - ), - InputParam.template( - "width", - description=( - "Width of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for " - "`text2image`, which works best at 8 megapixels. An edit defaults to the first image's size, at " - "least `canvas_min_pixels`, and works best at 4 or 8 megapixels." - ), - ), - ] _workflow_map = { "text2image": {"prompt": True}, "image_conditioned": {"prompt": True, "image": True}, diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py index e7016a0d5ed0..c2e0248f6410 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py @@ -122,13 +122,13 @@ def description(self): # before_denoise: inpainting class StableDiffusionXLInpaintBeforeDenoiseStep(SequentialPipelineBlocks): block_classes = [ - StableDiffusionXLImg2ImgSetTimestepsStep, + # set_timesteps is shared with img2img and defaults `strength` to 0.3; inpainting starts from (almost) pure + # noise + StableDiffusionXLImg2ImgSetTimestepsStep().new(inputs=[InputParam.template("strength", default=0.9999)]), StableDiffusionXLInpaintPrepareLatentsStep, StableDiffusionXLImg2ImgPrepareAdditionalConditioningStep, ] block_names = ["set_timesteps", "prepare_latents", "prepare_add_cond"] - # set_timesteps is shared with img2img and declares 0.3; inpainting starts from (almost) pure noise - input_overrides = [InputParam.template("strength", default=0.9999)] @property def description(self): diff --git a/src/diffusers/modular_pipelines/wan/encoders.py b/src/diffusers/modular_pipelines/wan/encoders.py index 9102d2f0e853..ddb6ba4e4b34 100644 --- a/src/diffusers/modular_pipelines/wan/encoders.py +++ b/src/diffusers/modular_pipelines/wan/encoders.py @@ -311,8 +311,8 @@ def description(self) -> str: def inputs(self) -> list[InputParam]: return [ InputParam("image", type_hint=PIL.Image.Image, required=True), - InputParam("height", type_hint=int, default=480), - InputParam("width", type_hint=int, default=832), + InputParam("height", type_hint=int, default=480, description="The target height of the video."), + InputParam("width", type_hint=int, default=832, description="The target width of the video."), ] @property @@ -505,7 +505,9 @@ def inputs(self) -> list[InputParam]: InputParam("resized_image", type_hint=PIL.Image.Image, required=True), InputParam("height"), InputParam("width"), - InputParam("num_frames", type_hint=int, default=81), + InputParam( + "num_frames", type_hint=int, default=81, description="The number of frames in the generated video." + ), InputParam("generator"), ] @@ -626,7 +628,9 @@ def inputs(self) -> list[InputParam]: ), InputParam("height"), InputParam("width"), - InputParam("num_frames", type_hint=int, default=81), + InputParam( + "num_frames", type_hint=int, default=81, description="The number of frames in the generated video." + ), InputParam("generator"), ] @@ -966,7 +970,9 @@ def inputs(self) -> list[InputParam]: InputParam("resized_last_image", type_hint=PIL.Image.Image, required=True), InputParam("height"), InputParam("width"), - InputParam("num_frames", type_hint=int, default=81), + InputParam( + "num_frames", type_hint=int, default=81, description="The number of frames in the generated video." + ), InputParam("generator"), ] diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py index e3bc4583c90e..32db1dfa02c1 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py @@ -14,7 +14,7 @@ from ...utils import logging from ..modular_pipeline import SequentialPipelineBlocks -from ..modular_pipeline_utils import InputParam, OutputParam +from ..modular_pipeline_utils import OutputParam from .before_denoise import ( WanAdditionalInputsStep, WanPrepareLatentsStep, @@ -54,11 +54,11 @@ class WanImage2VideoVaeEncoderStep(SequentialPipelineBlocks): image (`Image`): TODO: Add description. height (`int`, *optional*, defaults to 480): - TODO: Add description. + The target height of the video. width (`int`, *optional*, defaults to 832): - TODO: Add description. + The target width of the video. num_frames (`int`, *optional*, defaults to 81): - TODO: Add description. + The number of frames in the generated video. generator (`None`, *optional*): TODO: Add description. @@ -192,9 +192,9 @@ class Wan22Image2VideoBlocks(SequentialPipelineBlocks): image (`Image`): TODO: Add description. height (`int`, *optional*, defaults to 480): - TODO: Add description. + The target height of the video. width (`int`, *optional*, defaults to 832): - TODO: Add description. + The target width of the video. num_frames (`int`, *optional*, defaults to 81): The number of frames in the generated video. generator (`None`, *optional*): @@ -234,11 +234,6 @@ class Wan22Image2VideoBlocks(SequentialPipelineBlocks): "denoise", "decode", ] - # `vae_encoder` defaults `num_frames` to 81 and the denoise step declares it without a default (it fills it in - # from the latents or the components), so the pipeline declares it itself - input_overrides = [ - InputParam("num_frames", type_hint=int, default=81, description="The number of frames in the generated video.") - ] @property def description(self): diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py index 6e8308bf5bdc..26a09b0f9c07 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py @@ -14,7 +14,7 @@ from ...utils import logging from ..modular_pipeline import AutoPipelineBlocks, SequentialPipelineBlocks -from ..modular_pipeline_utils import InputParam, OutputParam +from ..modular_pipeline_utils import OutputParam from .before_denoise import ( WanAdditionalInputsStep, WanPrepareLatentsStep, @@ -58,9 +58,9 @@ class WanImage2VideoImageEncoderStep(SequentialPipelineBlocks): image (`Image`): TODO: Add description. height (`int`, *optional*, defaults to 480): - TODO: Add description. + The target height of the video. width (`int`, *optional*, defaults to 832): - TODO: Add description. + The target width of the video. Outputs: resized_image (`Image`): @@ -96,9 +96,9 @@ class WanFLF2VImageEncoderStep(SequentialPipelineBlocks): image (`Image`): TODO: Add description. height (`int`, *optional*, defaults to 480): - TODO: Add description. + The target height of the video. width (`int`, *optional*, defaults to 832): - TODO: Add description. + The target width of the video. last_image (`Image`): The last frameimage @@ -140,9 +140,9 @@ class WanAutoImageEncoderStep(AutoPipelineBlocks): image (`Image`, *optional*, depends on the workflow): TODO: Add description. height (`int`, *optional*, defaults to 480): - TODO: Add description. + The target height of the video. width (`int`, *optional*, defaults to 832): - TODO: Add description. + The target width of the video. last_image (`Image`, *optional*, depends on the workflow): The last frameimage @@ -194,11 +194,11 @@ class WanImage2VideoVaeEncoderStep(SequentialPipelineBlocks): image (`Image`): TODO: Add description. height (`int`, *optional*, defaults to 480): - TODO: Add description. + The target height of the video. width (`int`, *optional*, defaults to 832): - TODO: Add description. + The target width of the video. num_frames (`int`, *optional*, defaults to 81): - TODO: Add description. + The number of frames in the generated video. generator (`None`, *optional*): TODO: Add description. @@ -238,13 +238,13 @@ class WanFLF2VVaeEncoderStep(SequentialPipelineBlocks): image (`Image`): TODO: Add description. height (`int`, *optional*, defaults to 480): - TODO: Add description. + The target height of the video. width (`int`, *optional*, defaults to 832): - TODO: Add description. + The target width of the video. last_image (`Image`): The last frameimage num_frames (`int`, *optional*, defaults to 81): - TODO: Add description. + The number of frames in the generated video. generator (`None`, *optional*): TODO: Add description. @@ -293,13 +293,13 @@ class WanAutoVaeEncoderStep(AutoPipelineBlocks): image (`Image`, *optional*, depends on the workflow): TODO: Add description. height (`int`, *optional*, defaults to 480): - TODO: Add description. + The target height of the video. width (`int`, *optional*, defaults to 832): - TODO: Add description. + The target width of the video. last_image (`Image`, *optional*, depends on the workflow): The last frameimage num_frames (`int`, *optional*, defaults to 81): - TODO: Add description. + The number of frames in the generated video. generator (`None`, *optional*): TODO: Add description. @@ -498,17 +498,6 @@ class WanImage2VideoAutoBlocks(SequentialPipelineBlocks): "denoise", "decode", ] - # `image_encoder` and `vae_encoder` default `height`/`width` to 480/832 (the target area, the image sets the - # aspect ratio) and `num_frames` to 81, while the denoise step declares them without a default (it fills them in - # from the latents or the components). Both encoders are auto blocks the walk cannot count on, so the pipeline - # declares the three itself - input_overrides = [ - InputParam("height", type_hint=int, default=480, description="The target height of the video."), - InputParam("width", type_hint=int, default=832, description="The target width of the video."), - InputParam( - "num_frames", type_hint=int, default=81, description="The number of frames in the generated video." - ), - ] _workflow_map = { "image2video": {"image": True, "prompt": True}, diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py index d54cfd8f0186..6a3ea60df529 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py @@ -14,7 +14,7 @@ from ...utils import logging from ..modular_pipeline import SequentialPipelineBlocks -from ..modular_pipeline_utils import InputParam, OutputParam +from ..modular_pipeline_utils import OutputParam from .before_denoise import ( WanPrepareLatentsStep, WanSetTimestepsStep, @@ -196,11 +196,6 @@ class Wan22VaceBlocks(SequentialPipelineBlocks): "denoise", "decode", ] - # `vace_encoder` defaults `num_frames` to 81 and the denoise step declares it without a default (it fills it in - # from the latents or the components), so the pipeline declares it itself - input_overrides = [ - InputParam("num_frames", type_hint=int, default=81, description="The number of frames in the generated video.") - ] @property def description(self): diff --git a/tests/modular_pipelines/test_sequential_pipeline_blocks.py b/tests/modular_pipelines/test_sequential_pipeline_blocks.py index 718709cbcd92..303c8a44cd7c 100644 --- a/tests/modular_pipelines/test_sequential_pipeline_blocks.py +++ b/tests/modular_pipelines/test_sequential_pipeline_blocks.py @@ -13,7 +13,6 @@ # limitations under the License. import contextlib -import re import pytest @@ -36,14 +35,6 @@ logger = logging.get_logger("diffusers.modular_pipelines.modular_pipeline") -@contextlib.contextmanager -def warns(pattern): - """The body must log a warning matching `pattern`.""" - with CaptureLogger(logger) as logs: - yield - assert re.search(pattern, logs.out), logs.out - - @contextlib.contextmanager def quiet(): """The body must log nothing.""" @@ -617,20 +608,20 @@ def __call__(self, components, state): return FirstBlock, SecondBlock def test_two_explicit_values_first_wins(self): - # two explicit values for one shared input: the sequence keeps the first and says so + # two explicit values for one shared input: the sequence keeps the first (and logs the other at debug level) FirstBlock, SecondBlock = self._blocks(0.3, 0.9999) class Blocks(SequentialPipelineBlocks): block_classes = [FirstBlock, SecondBlock] block_names = ["first", "second"] - with warns("'second' declares default 0.9999 for input 'strength'.*keeps the earlier declaration"): + with quiet(): inputs = Blocks().inputs assert [inp.default for inp in inputs if inp.name == "strength"] == [0.3] - def test_none_after_a_value_is_not_a_conflict_a_value_after_none_is(self): - # a later block with no default takes whatever is set, nothing to say; a later block with a default that is not - # the one in use is told + def test_a_value_and_an_open_declaration_are_not_a_conflict_either_way(self): + # a later block with no default takes whatever is set; a later block with a default after an earlier block + # that leaves the input open has a fallback the sequence does not use. Neither is a conflict FirstBlock, SecondBlock = self._blocks(0.3, None) class Blocks(SequentialPipelineBlocks): @@ -638,7 +629,8 @@ class Blocks(SequentialPipelineBlocks): block_names = ["first", "second"] with quiet(): - Blocks().inputs + inputs = Blocks().inputs + assert [inp.default for inp in inputs if inp.name == "strength"] == [0.3] FirstBlock, SecondBlock = self._blocks(None, 0.3) @@ -646,22 +638,26 @@ class Blocks(SequentialPipelineBlocks): block_classes = [FirstBlock, SecondBlock] block_names = ["first", "second"] - with warns("'second' declares default 0.3 for input 'strength'"): - Blocks().inputs + with quiet(): + inputs = Blocks().inputs + # the sequence keeps the input open + assert [inp.default for inp in inputs if inp.name == "strength"] == [None] - def test_default_override_settles_the_conflict(self): + def test_new_on_a_sub_block_settles_the_conflict(self): FirstBlock, SecondBlock = self._blocks(0.3, 0.9999) class Blocks(SequentialPipelineBlocks): - block_classes = [FirstBlock, SecondBlock] + # the shared block is right elsewhere with 0.3; this sequence wants 0.9999 + block_classes = [FirstBlock().new(inputs=[InputParam("strength", default=0.9999)]), SecondBlock] block_names = ["first", "second"] - input_overrides = [InputParam("strength", default=0.9999)] with quiet(): inputs = Blocks().inputs assert [inp.default for inp in inputs if inp.name == "strength"] == [0.9999] + # the sub-block fills in the settled default at run time, it is its own declaration + assert [inp.default for inp in Blocks().sub_blocks["first"].inputs] == [0.9999] - def test_loop_input_without_default_is_a_conflict(self): + def test_loop_input_without_default_is_kept_open(self): FirstBlock, _ = self._blocks(0.9999, None) class Loop(LoopSequentialPipelineBlocks): @@ -673,12 +669,12 @@ class Loop(LoopSequentialPipelineBlocks): def loop_inputs(self): return [InputParam("timesteps", required=True), InputParam("strength")] - with warns("'first' declares default 0.9999 for input 'strength'"): + with quiet(): inputs = Loop().inputs - # the loop declaration comes first and wins + # the loop declaration comes first and wins; the sub-block's default is a fallback, not a conflict assert [inp.default for inp in inputs if inp.name == "strength"] == [None] - def test_loop_blocks_log_and_accept_the_override(self): + def test_loop_blocks_keep_the_earlier_declaration(self): FirstBlock, SecondBlock = self._blocks(0.3, 0.9999) class Loop(LoopSequentialPipelineBlocks): @@ -690,19 +686,20 @@ class Loop(LoopSequentialPipelineBlocks): def loop_inputs(self): return [InputParam("timesteps", required=True)] - with warns("'second' declares default 0.9999 for input 'strength'.*The loop keeps the earlier declaration"): - Loop().inputs + with quiet(): + inputs = Loop().inputs + assert [inp.default for inp in inputs if inp.name == "strength"] == [0.3] with quiet(): inputs = Loop.from_blocks_dict( - {"first": FirstBlock(), "second": SecondBlock()}, - input_overrides=[InputParam("strength", default=0.9999)], + {"first": FirstBlock().new(inputs=[InputParam("strength", default=0.9999)]), "second": SecondBlock()} ).inputs assert [inp.default for inp in inputs if inp.name == "strength"] == [0.9999] - def test_value_after_branch_defaults_is_a_conflict_the_reverse_is_not(self): - # the running branch fills in its default first, so a later leaf default is never the one in use: the composite - # is told. A conditional block after a value declares fallbacks for when the input is still unset, nothing to say + def test_value_after_branch_defaults_keeps_the_branch_defaults(self): + # the running branch fills in its default first, so a later leaf default is never the one in use and the + # per-branch record stays. A conditional block after a value declares fallbacks for when the input is still + # unset: the value stays FirstBlock, _ = self._blocks(0.5, None) Auto = strength_auto_blocks(0.3, 0.9999) @@ -710,8 +707,11 @@ class AutoThenValue(SequentialPipelineBlocks): block_classes = [Auto, FirstBlock] block_names = ["auto", "first"] - with warns("'first' declares default 0.5 .* declares per-branch defaults"): - AutoThenValue().inputs + with quiet(): + inputs = AutoThenValue().inputs + assert [(inp.default, inp.defaults_by_block) for inp in inputs if inp.name == "strength"] == [ + (None, {"auto.inpaint": 0.9999, "auto.img2img": 0.3}) + ] class ValueThenAuto(SequentialPipelineBlocks): block_classes = [FirstBlock, Auto] @@ -748,10 +748,9 @@ class AutoThenAuto(SequentialPipelineBlocks): with quiet(): AutoThenAuto().inputs - def test_value_after_a_required_branch_is_a_conflict(self): + def test_value_after_a_required_branch_keeps_the_record(self): # one branch requires the input and the other does not declare it, so the auto block records - # {"img2img": REQUIRED} with default `None`; the rule compares defaults only, so a later 0.5 is told it is not - # the one in use (it is, in the branch that does not declare the input: a known imprecision of the rule) + # {"img2img": REQUIRED} with default `None`; a later 0.5 does not change that FirstBlock, _ = self._blocks(0.5, None) Auto = strength_auto_blocks(REQUIRED, ...) @@ -759,7 +758,7 @@ class Blocks(SequentialPipelineBlocks): block_classes = [Auto, FirstBlock] block_names = ["auto", "first"] - with warns("'first' declares default 0.5 .* per-branch defaults {'auto.img2img': required}"): + with quiet(): (strength,) = [inp for inp in Blocks().inputs if inp.name == "strength"] assert (strength.required, strength.default, strength.defaults_by_block) == ( False, @@ -810,88 +809,77 @@ def __call__(self, components, state): return components, state -class TestOverrides: - def _sequential(self, **overrides): - class Blocks(SequentialPipelineBlocks): - block_classes = [PrepareLatentsBlock, UNetBlock] - block_names = ["prepare_latents", "unet"] +class TestNew: + """`ModularPipelineBlocks.new()`: a copy of a block with some of its declarations replaced.""" - for name, value in overrides.items(): - setattr(Blocks, name, value) - return Blocks() - - def test_input_override_replaces_the_derived_declaration(self): - blocks = self._sequential(input_overrides=[InputParam("batch_size", default=1)]) - (batch_size,) = [inp for inp in blocks.inputs if inp.name == "batch_size"] - assert batch_size.default == 1 - assert batch_size.required is False - assert "batch_size" not in blocks.required_inputs - - def test_override_does_not_mutate_the_sub_block_declaration(self): - blocks = self._sequential(input_overrides=[InputParam("batch_size", default=1)]) - _ = blocks.inputs - (batch_size,) = [inp for inp in blocks.sub_blocks["prepare_latents"].inputs if inp.name == "batch_size"] - assert batch_size.required is True - assert batch_size.default is None - - def test_component_and_config_overrides(self): - blocks = self._sequential( - component_overrides=[ComponentSpec("unet", description="any UNet2DConditionModel")], - config_overrides=[ConfigSpec("scale", default=16)], + def test_replaces_the_listed_declarations_and_leaves_the_block_alone(self): + block = PrepareLatentsBlock() + copy_ = block.new( + description="prepares latents for a fixed batch", + inputs=[InputParam("batch_size", default=1)], + intermediate_outputs=[OutputParam("latents", description="initial latents")], ) - assert [c.description for c in blocks.expected_components if c.name == "unet"] == ["any UNet2DConditionModel"] - assert [c.default for c in blocks.expected_configs if c.name == "scale"] == [16] - def test_unknown_name_raises(self): - blocks = self._sequential(input_overrides=[InputParam("strenght", default=0.5)]) - with pytest.raises(ValueError, match="strenght"): - blocks.inputs - blocks = self._sequential(component_overrides=[ComponentSpec("vae", description="x")]) - with pytest.raises(ValueError, match="vae"): - blocks.expected_components - - def test_entry_that_is_not_a_declaration_raises(self): - # a partial dict, or the `{name: declaration}` shape, is not a list of declarations - for overrides in ([{"default": 1}], {"batch_size": InputParam("batch_size", default=1)}): - blocks = self._sequential(input_overrides=overrides) - with pytest.raises(TypeError, match="must be a list of declarations"): - blocks.inputs - blocks = self._sequential(input_overrides=[ConfigSpec("batch_size", default=1)]) - with pytest.raises(TypeError, match="must be of type InputParam"): - blocks.inputs - - def test_name_declared_twice_raises(self): - blocks = self._sequential( - input_overrides=[InputParam("batch_size", default=1), InputParam("batch_size", default=2)] + assert copy_.description == "prepares latents for a fixed batch" + (batch_size,) = [inp for inp in copy_.inputs if inp.name == "batch_size"] + assert batch_size.default == 1 and batch_size.required is False + assert "batch_size" not in copy_.required_inputs + assert [out.description for out in copy_.intermediate_outputs if out.name == "latents"] == ["initial latents"] + # the other entries are the block's own + assert [inp.name for inp in copy_.inputs] == [inp.name for inp in block.inputs] + # the block itself is untouched, and still the class the copy derives from + (batch_size,) = [inp for inp in block.inputs if inp.name == "batch_size"] + assert batch_size.required is True and batch_size.default is None + assert isinstance(copy_, PrepareLatentsBlock) and type(copy_).__name__ == "PrepareLatentsBlock" + + def test_components_configs_and_outputs(self): + block = UNetBlock() + copy_ = block.new( + components=[ComponentSpec("unet", description="any UNet2DConditionModel")], + configs=[ConfigSpec("scale", default=16)], + outputs=[OutputParam("latents", description="the final latents")], ) - with pytest.raises(ValueError, match="more than once"): - blocks.inputs + assert [c.description for c in copy_.expected_components if c.name == "unet"] == ["any UNet2DConditionModel"] + assert [c.default for c in copy_.expected_configs if c.name == "scale"] == [16] + assert [out.description for out in copy_.outputs if out.name == "latents"] == ["the final latents"] + assert [c.default for c in block.expected_configs if c.name == "scale"] == [8] - def test_from_blocks_dict_accepts_overrides(self): - blocks = SequentialPipelineBlocks.from_blocks_dict( - {"prepare_latents": PrepareLatentsBlock(), "unet": UNetBlock()}, - input_overrides=[InputParam("batch_size", default=2)], - config_overrides=[ConfigSpec("scale", default=16)], - ) - assert [inp.default for inp in blocks.inputs if inp.name == "batch_size"] == [2] - assert [c.default for c in blocks.expected_configs if c.name == "scale"] == [16] + def test_keeps_what_the_block_was_created_with(self): + class RenamingBlock(ModularPipelineBlocks): + model_name = "test" - loop = LoopSequentialPipelineBlocks.from_blocks_dict( - {"unet": UNetBlock()}, - component_overrides=[ComponentSpec("unet", description="any UNet2DConditionModel")], - ) - assert [c.description for c in loop.expected_components if c.name == "unet"] == ["any UNet2DConditionModel"] + def __init__(self, latents_name="latents"): + super().__init__() + self.latents_name = latents_name - def test_auto_blocks_override_replaces_the_per_block_defaults(self): - Auto = strength_auto_blocks(0.3, 0.9999) + @property + def inputs(self): + return [InputParam(self.latents_name), InputParam("strength", default=0.3)] - (strength,) = [inp for inp in Auto().inputs if inp.name == "strength"] - assert strength.default is None - assert strength.defaults_by_block == {"inpaint": 0.9999, "img2img": 0.3} + def __call__(self, components, state): + return components, state - class AutoWithDefault(Auto): - input_overrides = [InputParam("strength", default=0.5)] + copy_ = RenamingBlock(latents_name="image_latents").new(inputs=[InputParam("strength", default=0.9)]) + assert [(inp.name, inp.default) for inp in copy_.inputs] == [("image_latents", None), ("strength", 0.9)] + + def test_unknown_name_raises(self): + with pytest.raises(ValueError, match="does not declare an input 'strenght'"): + PrepareLatentsBlock().new(inputs=[InputParam("strenght", default=0.5)]) + with pytest.raises(ValueError, match="does not declare a component 'vae'"): + UNetBlock().new(components=[ComponentSpec("vae", description="x")]) + + def test_a_block_with_sub_blocks_raises(self): + class Blocks(SequentialPipelineBlocks): + block_classes = [PrepareLatentsBlock, UNetBlock] + block_names = ["prepare_latents", "unet"] + + with pytest.raises(TypeError, match="derives its declarations from its sub-blocks"): + Blocks().new(inputs=[InputParam("batch_size", default=1)]) + + class Loop(LoopSequentialPipelineBlocks): + model_name = "test" + block_classes = [UNetBlock] + block_names = ["unet"] - (strength,) = [inp for inp in AutoWithDefault().inputs if inp.name == "strength"] - assert strength.default == 0.5 - assert strength.defaults_by_block is None + with pytest.raises(TypeError, match="derives its declarations from its sub-blocks"): + Loop().new(configs=[ConfigSpec("scale", default=16)]) From cecf77c7266afa2c53c183cc743fddcf8fec2aa2 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sun, 15 Nov 2026 05:24:21 +0100 Subject: [PATCH 14/33] [modular] `get_execution_blocks` keeps the outputs of the full blocks The extracted blocks listed every intermediate value as an output; they now list what the full blocks list, for the outputs the selected blocks produce. `from_blocks_dict` takes `outputs=`, and its `description=` works (it set a read-only property). Co-Authored-By: Claude Fable 5.1 --- .../sequential_pipeline_blocks.md | 4 +- .../modular_pipelines/modular_pipeline.py | 31 ++++++++++-- .../test_sequential_pipeline_blocks.py | 49 +++++++++++++++++++ 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md index a6dc22eb0771..c472aa06cf52 100644 --- a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md +++ b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md @@ -138,6 +138,6 @@ A few properties you set yourself: - `description`: We recommend adding a description for the assembled block to explain what the combined step does. - `model_name`: This is automatically derived from the sub-blocks but isn't always correct, so you may need to override it. -- `outputs`: By default this is the same as `intermediate_outputs`. Set it to list only the values the step is for, so the doc shows the final outputs instead of every intermediate value. Everything stays in the pipeline state either way. +- `outputs`: By default this is the same as `intermediate_outputs`. Set it to list only the values the step is for, so the doc shows the final outputs instead of every intermediate value. Everything stays in the pipeline state either way. The blocks `get_execution_blocks` and `get_workflow` return keep this list too, minus any output their selected blocks don't produce. -These properties, together with the aggregated `inputs`, `intermediate_outputs`, and `expected_components`, are used to automatically generate the `doc` property. \ No newline at end of file +These properties, together with the aggregated `inputs`, `intermediate_outputs`, and `expected_components`, are used to automatically generate the `doc` property. diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 12b82d714969..b5501fab46b1 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -1244,7 +1244,8 @@ class SequentialPipelineBlocks(ModularPipelineBlocks): @property def description(self): - return "" + # default to empty string; set by `from_blocks_dict(description=...)` or overridden in child classes + return self._description if self._description is not None else "" @property def model_name(self): @@ -1303,12 +1304,15 @@ def from_blocks_dict( cls, blocks_dict: dict[str, Any], description: str | None = None, + outputs: list[OutputParam] | None = None, ) -> "SequentialPipelineBlocks": """Creates a SequentialPipelineBlocks instance from a dictionary of blocks. Args: blocks_dict: Dictionary mapping block names to block classes or instances description: Description of the new block + outputs: the outputs the new block lists in its docs, in place of every intermediate output. Each must be + an intermediate output of one of the blocks Returns: A new SequentialPipelineBlocks instance @@ -1328,11 +1332,22 @@ def from_blocks_dict( instance.sub_blocks = sub_blocks if description is not None: - instance.description = description + instance._description = description + if outputs is not None: + produced = {out.name for out in instance.intermediate_outputs} + unknown = sorted(out.name for out in outputs if out.name not in produced) + if unknown: + raise ValueError( + f"`from_blocks_dict(outputs=...)`: {unknown} are not intermediate outputs of the given blocks." + ) + instance._outputs = list(outputs) return instance def __init__(self): + # set by `from_blocks_dict(description=..., outputs=...)`, `None` otherwise + self._description = None + self._outputs = None sub_blocks = InsertableDict() for block_name, block in zip(self.block_names, self.block_classes): if inspect.isclass(block): @@ -1443,10 +1458,11 @@ def intermediate_outputs(self) -> list[str]: return [replace(out, _maybe=out.name not in always) for out in combined_outputs] # `outputs` is what `doc` and the model card list as the block's outputs. By default that is every intermediate - # output; a composite overrides it to show only what the step is for, e.g. just `latents` for the `denoise` block. + # output; a composite overrides it to show only what the step is for, e.g. just `latents` for the `denoise` block, + # either by overriding the property in a child class or with `from_blocks_dict(outputs=...)`. @property def outputs(self) -> list[str]: - return self.intermediate_outputs + return self._outputs if self._outputs is not None else self.intermediate_outputs @torch.no_grad() def __call__(self, pipeline, state: PipelineState) -> tuple["ModularPipeline", PipelineState]: @@ -1536,7 +1552,12 @@ def fn_recursive_traverse(block, block_name, active_inputs): nested_blocks = fn_recursive_traverse(block, block_name, active_inputs) all_blocks.update(nested_blocks) - return SequentialPipelineBlocks.from_blocks_dict(all_blocks) + # if `outputs` is set on the composite block, it applies to the selected execution blocks as well, minus the + # ones they don't produce + produced = {out.name for block in all_blocks.values() for out in block.intermediate_outputs} + return SequentialPipelineBlocks.from_blocks_dict( + all_blocks, outputs=[out for out in self.outputs if out.name in produced] + ) def __repr__(self): class_name = self.__class__.__name__ diff --git a/tests/modular_pipelines/test_sequential_pipeline_blocks.py b/tests/modular_pipelines/test_sequential_pipeline_blocks.py index 303c8a44cd7c..f76ec130942a 100644 --- a/tests/modular_pipelines/test_sequential_pipeline_blocks.py +++ b/tests/modular_pipelines/test_sequential_pipeline_blocks.py @@ -883,3 +883,52 @@ class Loop(LoopSequentialPipelineBlocks): with pytest.raises(TypeError, match="derives its declarations from its sub-blocks"): Loop().new(configs=[ConfigSpec("scale", default=16)]) + + +class TestFromBlocksDict: + def test_description_and_outputs(self): + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"prepare_latents": PrepareLatentsBlock(), "unet": UNetBlock()}, + description="prepares and denoises", + outputs=[OutputParam("latents", description="denoised latents")], + ) + assert blocks.description == "prepares and denoises" + assert [out.name for out in blocks.outputs] == ["latents"] + # without them: no description, every intermediate output + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"prepare_latents": PrepareLatentsBlock(), "unet": UNetBlock()} + ) + assert blocks.description == "" + assert [out.name for out in blocks.outputs] == [out.name for out in blocks.intermediate_outputs] + + def test_an_output_no_block_produces_raises(self): + with pytest.raises(ValueError, match="\\['images'\\] are not intermediate outputs"): + SequentialPipelineBlocks.from_blocks_dict({"unet": UNetBlock()}, outputs=[OutputParam("images")]) + + def test_execution_blocks_keep_the_outputs_of_the_full_blocks(self): + class DecodeBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("latents", required=True)] + + @property + def intermediate_outputs(self): + return [OutputParam("images")] + + def __call__(self, components, state): + return components, state + + class Blocks(SequentialPipelineBlocks): + block_classes = [strength_auto_blocks(0.3, 0.9999), UNetBlock, DecodeBlock] + block_names = ["auto", "unet", "decode"] + + @property + def outputs(self): + # what the pipeline is for, plus one that only the img2img branch produces + return [OutputParam("images"), OutputParam("image_latents")] + + workflow = Blocks().get_execution_blocks(mask=True) + assert [out.name for out in workflow.outputs] == ["images"] + assert len(workflow.intermediate_outputs) > 1 From 71336b53bcf7147568be00e9345bf696c10a1fe9 Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Mon, 26 Oct 2026 05:56:47 +0000 Subject: [PATCH 15/33] add tests for checking optional and required inputs for modular --- .../modular_pipelines/testing_utils/common.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/modular_pipelines/testing_utils/common.py b/tests/modular_pipelines/testing_utils/common.py index 3ea912a6db5c..e0c472d7c52f 100644 --- a/tests/modular_pipelines/testing_utils/common.py +++ b/tests/modular_pipelines/testing_utils/common.py @@ -206,18 +206,9 @@ class ModularPipelineTesterMixin(BaseModularPipelineOutputMixin): """ def test_pipeline_call_signature(self): - pipe = self.get_pipeline() - input_parameters = pipe.blocks.input_names - optional_parameters = pipe.default_call_parameters - - def _check_for_parameters(parameters, expected_parameters, param_type): - remaining_parameters = {param for param in parameters if param not in expected_parameters} - assert len(remaining_parameters) == 0, ( - f"Required {param_type} parameters not present: {remaining_parameters}" - ) - - _check_for_parameters(self.params, input_parameters, "input") - _check_for_parameters(self.optional_params, optional_parameters, "optional") + blocks = self.pipeline_blocks_class() + input_parameters = set(blocks.input_names) + assert self.params <= input_parameters, self.params - input_parameters unsupported_parameters = {param for param in self.not_params if param in input_parameters} assert len(unsupported_parameters) == 0, ( @@ -234,6 +225,23 @@ def test_no_input_is_both_required_and_has_a_default(self): blocks.inputs assert "is required and declares a default" not in cap_logger.out, cap_logger.out + def test_pipeline_optional_parameters(self): + blocks = self.pipeline_blocks_class() + optional_parameters = {param.name for param in blocks.inputs if param.name is not None and not param.required} + assert self.optional_params <= optional_parameters, self.optional_params - optional_parameters + + def test_pipeline_required_inputs(self): + blocks_to_check = [self.pipeline_blocks_class()] + for blocks in blocks_to_check: + required_inputs = set(blocks.required_inputs) + for param in blocks.inputs: + if param.name is None: + continue + assert param.required == (param.name in required_inputs), (type(blocks).__name__, param) + if param.required: + assert param.default is None, (type(blocks).__name__, param) + blocks_to_check.extend(blocks.sub_blocks.values()) + def test_inference_batch_consistent(self, batch_sizes=[2], batch_generator=True): pipe = self.get_pipeline().to(torch_device) From 5d9d639e0520429451f4199278c5f52d1d064ba2 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 16 Nov 2026 18:22:21 +0100 Subject: [PATCH 16/33] [modular tests] require the canonical required inputs and that required means required `required_params` (default `prompt`) must be marked required on the pipeline blocks, as `optional_params` must be optional; `generator`, `height` and `width` join the optional set. A new test drops each required input in turn and expects the block that needs it to raise. Kandinsky S2V requires `audio` and `image` instead of a prompt. Co-Authored-By: Claude Fable 5.1 --- .../test_modular_pipeline_kandinsky_s2v.py | 1 + .../modular_pipelines/testing_utils/common.py | 38 +++++++++++++++++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py b/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py index 8e665f82f21f..c2c803e9a16b 100644 --- a/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py +++ b/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py @@ -73,6 +73,7 @@ class KandinskyS2VModularPipelineTesterConfig(BaseModularPipelineTesterConfig): params = frozenset(["image", "audio", "prompt", "height", "width", "num_latent_frames"]) batch_params = frozenset(["prompt"]) optional_params = frozenset(["num_inference_steps", "output_type"]) + required_params = frozenset(["audio", "image"]) output_name = "videos" latents_output_name = "latent_chunks" # One chunk, channels last: (num_latent_frames, latent_height, latent_width, channels). diff --git a/tests/modular_pipelines/testing_utils/common.py b/tests/modular_pipelines/testing_utils/common.py index e0c472d7c52f..3139dd1d8b1a 100644 --- a/tests/modular_pipelines/testing_utils/common.py +++ b/tests/modular_pipelines/testing_utils/common.py @@ -44,7 +44,12 @@ class BaseModularPipelineTesterConfig: # Canonical parameters that are passed to `__call__` regardless of the type of pipeline. They are always # optional and have common sense default values. - optional_params = frozenset(["num_inference_steps", "num_images_per_prompt", "latents", "output_type"]) + optional_params = frozenset( + ["num_inference_steps", "num_images_per_prompt", "latents", "output_type", "generator", "height", "width"] + ) + # Canonical parameters the pipeline cannot run without. `test_pipeline_required_parameters` asserts the blocks mark + # them required; a pipeline that does not take a prompt overrides this with its own, e.g. `{"audio", "image"}` + required_params = frozenset(["prompt"]) # Parameters the pipeline deliberately does NOT accept — e.g. `negative_prompt` on a # guidance-distilled pipeline. `test_pipeline_call_signature` asserts they are absent, # so accidentally (re)introducing one fails the test. @@ -206,6 +211,7 @@ class ModularPipelineTesterMixin(BaseModularPipelineOutputMixin): """ def test_pipeline_call_signature(self): + # test all the `params` are present in the pipeline inputs and `not_params` are not present blocks = self.pipeline_blocks_class() input_parameters = set(blocks.input_names) assert self.params <= input_parameters, self.params - input_parameters @@ -216,9 +222,9 @@ def test_pipeline_call_signature(self): ) def test_no_input_is_both_required_and_has_a_default(self): - # `InputParam` warns and drops `required` when a declaration is both required and has a default (see the - # pitfalls list in `.ai/references/modular.md`). Building the blocks and reading their `inputs` constructs - # every `InputParam` of every sub-block, so a clean log means no block in this pipeline declares one + # `InputParam` warns and drops `required` when a declaration is both required and has a default, + # e.g. `InputParam("num_inference_steps", default=20, required=True)` (see the pitfalls list in + # `.ai/references/modular.md`). This test makes sure no block of an official pipeline declares one logger = logging.get_logger("diffusers.modular_pipelines.modular_pipeline_utils") with CaptureLogger(logger) as cap_logger: blocks = self.pipeline_blocks_class() @@ -226,11 +232,23 @@ def test_no_input_is_both_required_and_has_a_default(self): assert "is required and declares a default" not in cap_logger.out, cap_logger.out def test_pipeline_optional_parameters(self): + # test to make sure all `self.optional_params` are present in the pipeline inputs and are marked as optional. + # `optional_params` is the set of optional parameters common to all pipelines, e.g. "num_inference_steps", + # "num_images_per_prompt", "latents", "output_type", "generator", "height", "width"; a subclass can override it blocks = self.pipeline_blocks_class() optional_parameters = {param.name for param in blocks.inputs if param.name is not None and not param.required} assert self.optional_params <= optional_parameters, self.optional_params - optional_parameters + def test_pipeline_required_parameters(self): + # test to make sure all `self.required_params` are present in the pipeline inputs and are marked as required. + # `required_params` is "prompt" by default; a pipeline that does not take a prompt overrides it + blocks = self.pipeline_blocks_class() + required_parameters = {param.name for param in blocks.inputs if param.name is not None and param.required} + assert self.required_params <= required_parameters, self.required_params - required_parameters + def test_pipeline_required_inputs(self): + # this test makes sure the `inputs[].required` flag is consistent with the `required_inputs` property, on the + # pipeline blocks and on every sub-block. The two used to disagree on composite blocks, see #15009 blocks_to_check = [self.pipeline_blocks_class()] for blocks in blocks_to_check: required_inputs = set(blocks.required_inputs) @@ -242,6 +260,18 @@ def test_pipeline_required_inputs(self): assert param.default is None, (type(blocks).__name__, param) blocks_to_check.extend(blocks.sub_blocks.values()) + def test_required_inputs_are_required(self): + # this test makes sure all the inputs marked as required are indeed required: an input the blocks mark + # required is one the pipeline cannot run without, so leave each one out in turn and the block that needs it + # raises + pipe = self.get_pipeline().to(torch_device) + inputs = self.get_dummy_inputs() + required = [param.name for param in pipe.blocks.inputs if param.required] + assert set(required) <= set(inputs), set(required) - set(inputs) + for name in required: + with pytest.raises(ValueError, match=f"Required input '{name}' is missing"): + pipe(**{k: v for k, v in inputs.items() if k != name}, output=self.output_name) + def test_inference_batch_consistent(self, batch_sizes=[2], batch_generator=True): pipe = self.get_pipeline().to(torch_device) From f7b4b80f155de7927e9c29223e416bad75d933bf Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 16 Nov 2026 20:42:49 +0100 Subject: [PATCH 17/33] [modular] rewrap a docstring line in `new()` for doc-builder Co-Authored-By: Claude Fable 5.1 --- src/diffusers/modular_pipelines/modular_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index b5501fab46b1..ec984bde0e1c 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -444,8 +444,8 @@ def new( """ Creates a copy of this block with some of its declarations replaced, e.g. another default for an input, so the same block can be reused where only what it declares differs. The block's code is not changed and the copy - keeps what this block was created with. An entry cannot be added or removed: a block that reads other inputs - or runs other code is a block of its own. + keeps what this block was created with. An entry cannot be added or removed: a block that reads other inputs or + runs other code is a block of its own. Args: description: the description of the new block, in place of this block's From 359738099c69fc852e4e19c56d24cbfd13cc375a Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 16 Nov 2026 20:44:24 +0100 Subject: [PATCH 18/33] [modular] regenerate PAN 2 and MiniMax H3 Image docstrings after merging main Co-Authored-By: Claude Fable 5.1 --- .../modular_blocks_minimax_h3_image.py | 28 +++++++++---------- .../pan2/modular_blocks_pan2.py | 16 +++++------ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/diffusers/modular_pipelines/minimax_h3_image/modular_blocks_minimax_h3_image.py b/src/diffusers/modular_pipelines/minimax_h3_image/modular_blocks_minimax_h3_image.py index 86bc6e0d326b..df8dc51131c1 100644 --- a/src/diffusers/modular_pipelines/minimax_h3_image/modular_blocks_minimax_h3_image.py +++ b/src/diffusers/modular_pipelines/minimax_h3_image/modular_blocks_minimax_h3_image.py @@ -50,7 +50,7 @@ class MiniMaxH3ImageAutoBeforeEncodeStep(AutoPipelineBlocks): (default: 1024) vision_max_short_edge (default: 1024) Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, depends on the workflow): The reference image of the edit, or a list of them. The prompt refers to them in order as ``, ``, ... @@ -89,7 +89,7 @@ class MiniMaxH3ImageAutoTextEncoderStep(AutoPipelineBlocks): Inputs: prompt (`str`): The prompt to guide generation, a single string. - vision_images (`list`, *optional*): + vision_images (`list`, *optional*, depends on the workflow): The reference images resized for the text encoder. Outputs: @@ -127,7 +127,7 @@ class MiniMaxH3ImageAutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLMiniMaxH3Image`) Inputs: - reference_images (`list`, *optional*): + reference_images (`list`, *optional*, depends on the workflow): The reference images resized for the VAE. generator (`Generator`, *optional*): The generator of the request. One posterior sample per reference is drawn from it. @@ -321,9 +321,9 @@ class MiniMaxH3ImageAutoDenoiseStep(AutoPipelineBlocks): 0.23200887441635132, 0.0]): The sampling grid the checkpoint's per-step output heads were distilled on. Inputs: - height (`int`): + height (`int`, *optional*, defaults to 2048): Height of the returned image in pixels. - width (`int`): + width (`int`, *optional*, defaults to 2048): Width of the returned image in pixels. min_generation_area (`int`, *optional*, defaults to 4194304): The smallest area, in pixels, an image is generated at. A smaller `height x width` is generated on a @@ -331,12 +331,12 @@ class MiniMaxH3ImageAutoDenoiseStep(AutoPipelineBlocks): generate at the requested size. text_token_tags (`Tensor`): The modality tag of every row of `prompt_embeds`. - reference_latents (`list`, *optional*): + reference_latents (`list`, *optional*, depends on the workflow): The conditioning latents of the reference images. Their shape is where every reference's geometry comes from. generator (`Generator`, *optional*): Torch generator for deterministic generation. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noise of shape `(1, latent_channels, latent_height, latent_width)`, used instead of the draw. reference_noise (`list`, *optional*): @@ -389,29 +389,29 @@ class MiniMaxH3ImageBlocks(SequentialPipelineBlocks): per-step output heads were distilled on. Inputs: - image (`Image | list`, *optional*): + image (`Image | list`, *optional*, depends on the workflow): The reference image of the edit, or a list of them. The prompt refers to them in order as ``, ``, ... prompt (`str`): The prompt to guide generation, a single string. - vision_images (`list`, *optional*): + vision_images (`list`, *optional*, depends on the workflow): The reference images resized for the text encoder. - reference_images (`list`, *optional*): + reference_images (`list`, *optional*, depends on the workflow): The reference images resized for the VAE. generator (`Generator`, *optional*): The generator of the request. One posterior sample per reference is drawn from it. - height (`int`): + height (`int`, *optional*, defaults to 2048): Height of the returned image in pixels. - width (`int`): + width (`int`, *optional*, defaults to 2048): Width of the returned image in pixels. min_generation_area (`int`, *optional*, defaults to 4194304): The smallest area, in pixels, an image is generated at. A smaller `height x width` is generated on a canvas scaled up to this area and resized back to `height x width` after decoding. Pass 1024 to always generate at the requested size. - reference_latents (`list`, *optional*): + reference_latents (`list`, *optional*, depends on the workflow): The conditioning latents of the reference images. Their shape is where every reference's geometry comes from. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noise of shape `(1, latent_channels, latent_height, latent_width)`, used instead of the draw. reference_noise (`list`, *optional*): diff --git a/src/diffusers/modular_pipelines/pan2/modular_blocks_pan2.py b/src/diffusers/modular_pipelines/pan2/modular_blocks_pan2.py index f860f712202d..1bb5952c951d 100644 --- a/src/diffusers/modular_pipelines/pan2/modular_blocks_pan2.py +++ b/src/diffusers/modular_pipelines/pan2/modular_blocks_pan2.py @@ -42,7 +42,7 @@ class PAN2AutoVaeEncoderStep(AutoPipelineBlocks): vae (`AutoencoderKLWan`) video_processor (`VideoProcessor`) Inputs: - image (`Image`, *optional*): + image (`Image`, *optional*, depends on the workflow): The first frame, a single image. height (`int`, *optional*): The height in pixels of the generated video, 704 by default. @@ -209,15 +209,15 @@ class PAN2AutoCoreDenoiseStep(AutoPipelineBlocks): Text embeddings used to guide the video generation. negative_prompt_embeds (`Tensor`, *optional*): Negative text embeddings used to guide the video generation. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, depends on the workflow): The normalized first-frame latents. Can be generated from the vae_encoder step. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. num_frames (`int`, *optional*, defaults to 313): Number of video frames to generate, of the form `4 * n + 1`. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents to use as inputs for video generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. @@ -276,7 +276,7 @@ class PAN2Blocks(SequentialPipelineBlocks): Number of video frames to generate, of the form `4 * n + 1`. fps (`float`, *optional*, defaults to 24.0): The frame rate of the video, which sets its duration in the default negative prompt. - image (`Image`, *optional*): + image (`Image`, *optional*, depends on the workflow): The first frame, a single image. height (`int`, *optional*): The height in pixels of the generated video, 704 by default. @@ -284,13 +284,13 @@ class PAN2Blocks(SequentialPipelineBlocks): The width in pixels of the generated video, 1248 by default. num_videos_per_prompt (`int`, *optional*, defaults to 1): The number of videos to generate per prompt. - image_latents (`Tensor`, *optional*): + image_latents (`Tensor`, *optional*, depends on the workflow): The normalized first-frame latents. Can be generated from the vae_encoder step. - num_inference_steps (`int`): + num_inference_steps (`int`, *optional*, defaults to 50): The number of denoising steps. sigmas (`list`, *optional*): Custom sigmas for the denoising process. - latents (`Tensor`): + latents (`Tensor`, *optional*): Pre-generated noisy latents to use as inputs for video generation. generator (`Generator`, *optional*): Torch generator for deterministic generation. From 4aeab7b2d5f0700ff561a3a8ad4f4c90a41ae8bd Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 17 Nov 2026 01:32:35 +0100 Subject: [PATCH 19/33] [modular] the Flux, SD3, SDXL, Wan and Z-Image text encoders require `prompt` They declared it with `InputParam("prompt")`, so the pipelines listed it as optional and crashed on `None`. `InputParam.template("prompt")` marks it required and gives it the shared description. Wan Animate 2 and Wan VACE take one prompt per call, so their description says so; VACE wraps the shared Wan encoder with `new()` for it. Co-Authored-By: Claude Fable 5.1 --- src/diffusers/modular_pipelines/flux/encoders.py | 2 +- .../modular_pipelines/flux/modular_blocks_flux.py | 4 ++-- .../flux/modular_blocks_flux_kontext.py | 4 ++-- .../modular_pipelines/stable_diffusion_3/encoders.py | 5 +---- .../modular_blocks_stable_diffusion_3.py | 4 ++-- .../stable_diffusion_xl/encoders.py | 2 +- .../modular_blocks_stable_diffusion_xl.py | 4 ++-- src/diffusers/modular_pipelines/wan/encoders.py | 2 +- .../modular_pipelines/wan/modular_blocks_wan.py | 4 ++-- .../modular_pipelines/wan/modular_blocks_wan22.py | 4 ++-- .../wan/modular_blocks_wan22_i2v.py | 4 ++-- .../modular_pipelines/wan/modular_blocks_wan_i2v.py | 4 ++-- .../modular_pipelines/wan/modular_blocks_wan_vace.py | 12 ++++++++---- .../modular_pipelines/wan_animate_2/encoders.py | 2 +- .../wan_animate_2/modular_blocks_wan_animate_2.py | 2 +- .../modular_blocks_wan_animate_2_distilled.py | 2 +- src/diffusers/modular_pipelines/z_image/encoders.py | 2 +- .../z_image/modular_blocks_z_image.py | 4 ++-- 18 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/diffusers/modular_pipelines/flux/encoders.py b/src/diffusers/modular_pipelines/flux/encoders.py index fb7b40024b0a..19566217c674 100644 --- a/src/diffusers/modular_pipelines/flux/encoders.py +++ b/src/diffusers/modular_pipelines/flux/encoders.py @@ -307,7 +307,7 @@ def expected_components(self) -> list[ComponentSpec]: @property def inputs(self) -> list[InputParam]: return [ - InputParam("prompt"), + InputParam.template("prompt"), InputParam("prompt_2"), InputParam("max_sequence_length", type_hint=int, default=512, required=False), InputParam("joint_attention_kwargs"), diff --git a/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py b/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py index 64b9372e880b..7d56635bd3d8 100644 --- a/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py +++ b/src/diffusers/modular_pipelines/flux/modular_blocks_flux.py @@ -537,8 +537,8 @@ class FluxAutoBlocks(SequentialPipelineBlocks): (`FlowMatchEulerDiscreteScheduler`) transformer (`FluxTransformer2DModel`) Inputs: - prompt (`None`, *optional*): - TODO: Add description. + prompt (`str`): + The prompt or prompts to guide image generation. prompt_2 (`None`, *optional*): TODO: Add description. max_sequence_length (`int`, *optional*, defaults to 512): diff --git a/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py b/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py index ec8050d26147..43a403434be3 100644 --- a/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py +++ b/src/diffusers/modular_pipelines/flux/modular_blocks_flux_kontext.py @@ -546,8 +546,8 @@ class FluxKontextAutoBlocks(SequentialPipelineBlocks): (`FlowMatchEulerDiscreteScheduler`) transformer (`FluxTransformer2DModel`) Inputs: - prompt (`None`, *optional*): - TODO: Add description. + prompt (`str`): + The prompt or prompts to guide image generation. prompt_2 (`None`, *optional*): TODO: Add description. max_sequence_length (`int`, *optional*, defaults to 512): diff --git a/src/diffusers/modular_pipelines/stable_diffusion_3/encoders.py b/src/diffusers/modular_pipelines/stable_diffusion_3/encoders.py index ea163b3b9cf9..a7e176a6c86d 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_3/encoders.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_3/encoders.py @@ -479,10 +479,7 @@ def expected_components(self) -> list[ComponentSpec]: @property def inputs(self) -> list[InputParam]: return [ - InputParam( - "prompt", - description="The prompt or prompts to guide the image generation.", - ), + InputParam.template("prompt"), InputParam( "prompt_2", description="The prompt or prompts to be sent to tokenizer_2 and text_encoder_2.", diff --git a/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py b/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py index 716443369307..fb7ddcdd3e86 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_3/modular_blocks_stable_diffusion_3.py @@ -302,8 +302,8 @@ class StableDiffusion3AutoBlocks(SequentialPipelineBlocks): (`FlowMatchEulerDiscreteScheduler`) guider (`ClassifierFreeGuidance`) transformer (`SD3Transformer2DModel`) Inputs: - prompt (`None`, *optional*): - The prompt or prompts to guide the image generation. + prompt (`str`): + The prompt or prompts to guide image generation. prompt_2 (`None`, *optional*): The prompt or prompts to be sent to tokenizer_2 and text_encoder_2. prompt_3 (`None`, *optional*): diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py index 381badb2204d..72eb113e4f0a 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/encoders.py @@ -244,7 +244,7 @@ def expected_configs(self) -> list[ConfigSpec]: @property def inputs(self) -> list[InputParam]: return [ - InputParam("prompt"), + InputParam.template("prompt"), InputParam("prompt_2"), InputParam("negative_prompt"), InputParam("negative_prompt_2"), diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py index c2e0248f6410..3ac43f404764 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/modular_blocks_stable_diffusion_xl.py @@ -320,8 +320,8 @@ class StableDiffusionXLAutoBlocks(SequentialPipelineBlocks): force_zeros_for_empty_prompt (default: True) requires_aesthetics_score (default: False) Inputs: - prompt (`None`, *optional*): - TODO: Add description. + prompt (`str`): + The prompt or prompts to guide image generation. prompt_2 (`None`, *optional*): TODO: Add description. negative_prompt (`None`, *optional*): diff --git a/src/diffusers/modular_pipelines/wan/encoders.py b/src/diffusers/modular_pipelines/wan/encoders.py index ddb6ba4e4b34..ccfc29760542 100644 --- a/src/diffusers/modular_pipelines/wan/encoders.py +++ b/src/diffusers/modular_pipelines/wan/encoders.py @@ -178,7 +178,7 @@ def expected_components(self) -> list[ComponentSpec]: @property def inputs(self) -> list[InputParam]: return [ - InputParam("prompt"), + InputParam.template("prompt"), InputParam("negative_prompt"), InputParam("max_sequence_length", default=512), ] diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan.py index a664fff3a1c1..88037f24c2f1 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan.py @@ -111,8 +111,8 @@ class WanBlocks(SequentialPipelineBlocks): (`VideoProcessor`) Inputs: - prompt (`None`, *optional*): - TODO: Add description. + prompt (`str`): + The prompt or prompts to guide image generation. negative_prompt (`None`, *optional*): TODO: Add description. max_sequence_length (`None`, *optional*, defaults to 512): diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22.py index 24bf2a6341cd..3f491944bc6b 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22.py @@ -123,8 +123,8 @@ class Wan22Blocks(SequentialPipelineBlocks): noise stages. Inputs: - prompt (`None`, *optional*): - TODO: Add description. + prompt (`str`): + The prompt or prompts to guide image generation. negative_prompt (`None`, *optional*): TODO: Add description. max_sequence_length (`None`, *optional*, defaults to 512): diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py index 32db1dfa02c1..791c9d36cd07 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan22_i2v.py @@ -183,8 +183,8 @@ class Wan22Image2VideoBlocks(SequentialPipelineBlocks): noise stages. Inputs: - prompt (`None`, *optional*): - TODO: Add description. + prompt (`str`): + The prompt or prompts to guide image generation. negative_prompt (`None`, *optional*): TODO: Add description. max_sequence_length (`None`, *optional*, defaults to 512): diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py index 26a09b0f9c07..1feef98ba0d3 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_i2v.py @@ -441,8 +441,8 @@ class WanImage2VideoAutoBlocks(SequentialPipelineBlocks): (`UniPCMultistepScheduler`) Inputs: - prompt (`None`, *optional*): - TODO: Add description. + prompt (`str`): + The prompt or prompts to guide image generation. negative_prompt (`None`, *optional*): TODO: Add description. max_sequence_length (`None`, *optional*, defaults to 512): diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py index 6a3ea60df529..df8a138a529d 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_vace.py @@ -14,7 +14,7 @@ from ...utils import logging from ..modular_pipeline import SequentialPipelineBlocks -from ..modular_pipeline_utils import OutputParam +from ..modular_pipeline_utils import InputParam, OutputParam from .before_denoise import ( WanPrepareLatentsStep, WanSetTimestepsStep, @@ -137,8 +137,8 @@ class Wan22VaceBlocks(SequentialPipelineBlocks): noise stages. Inputs: - prompt (`None`, *optional*): - TODO: Add description. + prompt (`str`): + The prompt to guide generation, a single string. negative_prompt (`None`, *optional*): TODO: Add description. max_sequence_length (`None`, *optional*, defaults to 512): @@ -185,7 +185,11 @@ class Wan22VaceBlocks(SequentialPipelineBlocks): model_name = "wan-vace" block_classes = [ - WanTextEncoderStep, + # VACE takes one prompt per call (`WanVaceAdditionalInputsStep` raises on a list), the shared encoder says + # "prompt or prompts" + WanTextEncoderStep().new( + inputs=[InputParam.template("prompt", description="The prompt to guide generation, a single string.")] + ), WanVaceEncoderStep, Wan22VaceCoreDenoiseStep, WanVaeDecoderStep, diff --git a/src/diffusers/modular_pipelines/wan_animate_2/encoders.py b/src/diffusers/modular_pipelines/wan_animate_2/encoders.py index ffba4c672691..11d9ba40f0fe 100644 --- a/src/diffusers/modular_pipelines/wan_animate_2/encoders.py +++ b/src/diffusers/modular_pipelines/wan_animate_2/encoders.py @@ -141,7 +141,7 @@ def expected_components(self) -> list[ComponentSpec]: @property def inputs(self) -> list[InputParam]: return [ - InputParam.template("prompt"), + InputParam.template("prompt", description="The prompt to guide generation, a single string."), InputParam.template("negative_prompt"), InputParam( "prompt_ref", diff --git a/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2.py b/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2.py index f77eb378c15b..dd5aea09895d 100644 --- a/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2.py +++ b/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2.py @@ -261,7 +261,7 @@ class WanAnimate2Blocks(SequentialPipelineBlocks): Inputs: prompt (`str`): - The prompt or prompts to guide image generation. + The prompt to guide generation, a single string. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. prompt_ref (`str`, *optional*, defaults to 人物动作的参考视频): diff --git a/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2_distilled.py b/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2_distilled.py index 8eab815897da..c4d8ffc3290c 100644 --- a/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2_distilled.py +++ b/src/diffusers/modular_pipelines/wan_animate_2/modular_blocks_wan_animate_2_distilled.py @@ -263,7 +263,7 @@ class WanAnimate2DistilledBlocks(SequentialPipelineBlocks): Inputs: prompt (`str`): - The prompt or prompts to guide image generation. + The prompt to guide generation, a single string. negative_prompt (`str`, *optional*): The prompt or prompts not to guide the image generation. prompt_ref (`str`, *optional*, defaults to 人物动作的参考视频): diff --git a/src/diffusers/modular_pipelines/z_image/encoders.py b/src/diffusers/modular_pipelines/z_image/encoders.py index c3ab717078e1..ae1606ff3c73 100644 --- a/src/diffusers/modular_pipelines/z_image/encoders.py +++ b/src/diffusers/modular_pipelines/z_image/encoders.py @@ -148,7 +148,7 @@ def expected_components(self) -> list[ComponentSpec]: @property def inputs(self) -> list[InputParam]: return [ - InputParam("prompt"), + InputParam.template("prompt"), InputParam("negative_prompt"), InputParam("max_sequence_length", default=512), ] diff --git a/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py b/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py index 58253c903b62..93dbe4746cb4 100644 --- a/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py +++ b/src/diffusers/modular_pipelines/z_image/modular_blocks_z_image.py @@ -277,8 +277,8 @@ class ZImageAutoBlocks(SequentialPipelineBlocks): (`FlowMatchEulerDiscreteScheduler`) Inputs: - prompt (`None`, *optional*): - TODO: Add description. + prompt (`str`): + The prompt or prompts to guide image generation. negative_prompt (`None`, *optional*): TODO: Add description. max_sequence_length (`None`, *optional*, defaults to 512): From 0734863590004a8538821cef08fe20f9d4728a87 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 17 Nov 2026 01:33:39 +0100 Subject: [PATCH 20/33] [modular tests] Flux2, Ernie-Image and HunyuanVideo 1.5 require no input Their text encoders encode an empty prompt when `prompt` is not given, so the default `required_params` does not apply. Co-Authored-By: Claude Fable 5.1 --- .../ernie_image/test_modular_pipeline_ernie_image.py | 2 ++ tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py | 4 ++++ .../flux2/test_modular_pipeline_flux2_klein.py | 4 ++++ .../flux2/test_modular_pipeline_flux2_klein_base.py | 4 ++++ .../test_modular_pipeline_hunyuan_video1_5.py | 2 ++ 5 files changed, 16 insertions(+) diff --git a/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py b/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py index 0de66222be6e..04b1d56f86db 100644 --- a/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py +++ b/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py @@ -44,6 +44,8 @@ class ErnieImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "akshan-main/tiny-ernie-image-modular-pipe" expected_latents_shape = (1, 16, 2, 2) params = frozenset(["prompt", "height", "width"]) + # the text encoder encodes an empty prompt when none is given, so no input is required + required_params = frozenset() batch_params = frozenset(["prompt"]) optional_params = frozenset(["num_inference_steps", "num_images_per_prompt", "latents"]) expected_workflow_blocks = ERNIE_IMAGE_WORKFLOWS diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py index 6acd1a290f2e..f30e1ad0473f 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py @@ -55,6 +55,8 @@ class Flux2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-modular" expected_latents_shape = (1, 4, 16, 16) params = frozenset(["prompt", "height", "width", "guidance_scale"]) + # the text encoder encodes an empty prompt when none is given, so no input is required + required_params = frozenset() batch_params = frozenset(["prompt"]) expected_workflow_blocks = FLUX2_TEXT2IMAGE_WORKFLOWS @@ -116,6 +118,8 @@ class Flux2ImageConditionedModularPipelineTesterConfig(BaseModularPipelineTester pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-modular" expected_latents_shape = (1, 4, 16, 16) params = frozenset(["prompt", "height", "width", "guidance_scale", "image"]) + # the text encoder encodes an empty prompt when none is given, so no input is required + required_params = frozenset() batch_params = frozenset(["prompt", "image"]) expected_workflow_blocks = FLUX2_IMAGE_CONDITIONED_WORKFLOWS diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py index 93d5c818e056..8b75cab1ece9 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py @@ -54,6 +54,8 @@ class Flux2KleinModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-modular" expected_latents_shape = (1, 4, 16, 16) params = frozenset(["prompt", "height", "width"]) + # the text encoder encodes an empty prompt when none is given, so no input is required + required_params = frozenset() batch_params = frozenset(["prompt"]) not_params = frozenset(["negative_prompt"]) expected_workflow_blocks = FLUX2_KLEIN_WORKFLOWS @@ -114,6 +116,8 @@ class Flux2KleinImageConditionedModularPipelineTesterConfig(BaseModularPipelineT pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-modular" expected_latents_shape = (1, 4, 16, 16) params = frozenset(["prompt", "height", "width", "image"]) + # the text encoder encodes an empty prompt when none is given, so no input is required + required_params = frozenset() batch_params = frozenset(["prompt", "image"]) not_params = frozenset(["negative_prompt"]) expected_workflow_blocks = FLUX2_KLEIN_IMAGE_CONDITIONED_WORKFLOWS diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py index 90e45a781df8..f47f160fd62d 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py @@ -54,6 +54,8 @@ class Flux2KleinBaseModularPipelineTesterConfig(BaseModularPipelineTesterConfig) pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-base-modular" expected_latents_shape = (1, 4, 16, 16) params = frozenset(["prompt", "height", "width"]) + # the text encoder encodes an empty prompt when none is given, so no input is required + required_params = frozenset() batch_params = frozenset(["prompt"]) expected_workflow_blocks = FLUX2_KLEIN_BASE_WORKFLOWS @@ -113,6 +115,8 @@ class Flux2KleinBaseImageConditionedModularPipelineTesterConfig(BaseModularPipel pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-base-modular" expected_latents_shape = (1, 4, 16, 16) params = frozenset(["prompt", "height", "width", "image"]) + # the text encoder encodes an empty prompt when none is given, so no input is required + required_params = frozenset() batch_params = frozenset(["prompt", "image"]) expected_workflow_blocks = FLUX2_KLEIN_BASE_IMAGE_CONDITIONED_WORKFLOWS diff --git a/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py b/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py index 9ee9980f81e0..3fa9368931ad 100644 --- a/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py +++ b/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py @@ -55,6 +55,8 @@ class HunyuanVideo15ModularPipelineTesterConfig(BaseModularPipelineTesterConfig) pretrained_model_name_or_path = "akshan-main/tiny-hunyuanvideo1_5-modular-pipe" expected_latents_shape = (1, 32, 3, 2, 2) params = frozenset(["prompt", "height", "width", "num_frames"]) + # the text encoder encodes an empty prompt when none is given, so no input is required + required_params = frozenset() batch_params = frozenset(["prompt"]) optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) expected_workflow_blocks = HUNYUANVIDEO15_WORKFLOWS From 3e1bf4a7113d965b7b97204b96024119fe0a21cf Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 17 Nov 2026 01:33:39 +0100 Subject: [PATCH 21/33] [modular tests] `batch_params` defaults to `prompt`; the mixin skips the batch tests when it is empty Every pipeline with an empty `batch_params` raises on a list of prompts, which the comment next to it now says; the 32 per-class skip stubs are gone. Kandinsky S2V joins them. The unused `intermediate_params` is removed. Co-Authored-By: Claude Fable 5.1 --- .../cosmos/test_modular_pipeline_cosmos3.py | 9 +------- ...test_modular_pipeline_cosmos3_distilled.py | 9 +------- tests/modular_pipelines/echo/testing_utils.py | 1 - .../test_modular_pipeline_ernie_image.py | 1 - .../flux/test_modular_pipeline_flux.py | 1 - .../flux2/test_modular_pipeline_flux2.py | 1 - .../test_modular_pipeline_flux2_klein.py | 1 - .../test_modular_pipeline_flux2_klein_base.py | 1 - .../flux3/test_modular_pipeline_flux3.py | 17 +------------- .../helios/test_modular_pipeline_helios.py | 2 -- .../test_modular_pipeline_hunyuan_video1_5.py | 1 - .../test_modular_pipeline_kandinsky_s2v.py | 21 ++--------------- .../krea2/test_modular_pipeline_krea2.py | 1 - .../test_modular_pipeline_krea2_turbo.py | 1 - .../ltx/test_modular_pipeline_ltx.py | 1 - .../ltx2/test_modular_pipeline_ltx2.py | 1 - .../ltx2/test_modular_pipeline_ltx25.py | 1 - .../test_modular_pipeline_minimax_h3.py | 18 ++------------- .../test_modular_pipeline_minimax_h3_image.py | 9 +------- .../test_modular_pipeline_minimax_music3.py | 10 +------- .../test_modular_pipeline_p_h3_image.py | 9 +------- .../pan2/test_modular_pipeline_pan2.py | 9 +------- .../test_modular_pipeline_pika_h3_flex.py | 18 ++------------- ...test_modular_pipeline_qwenimage21_turbo.py | 1 - ...est_modular_pipeline_stable_diffusion_3.py | 1 - .../modular_pipelines/testing_utils/common.py | 23 ++++++------------- .../wan/test_modular_pipeline_wan.py | 1 - .../wan/test_modular_pipeline_wan_vace.py | 9 +------- .../test_modular_pipeline_wan_animate_2.py | 9 +------- .../z_image/test_modular_pipeline_z_image.py | 1 - 30 files changed, 22 insertions(+), 166 deletions(-) diff --git a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py index 5a2723b84625..cf7e2543a6eb 100644 --- a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py +++ b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py @@ -134,6 +134,7 @@ class Cosmos3OmniModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-cosmos3-modular-pipe" expected_latents_shape = (1, 4, 2, 4, 4) params = frozenset(["prompt", "height", "width", "num_frames", "guidance_scale"]) + # one prompt per call: the text encoder raises on a list batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "output_type"]) output_name = "videos" @@ -159,14 +160,6 @@ def get_dummy_inputs(self, seed=0): class TestCosmos3OmniModularPipelineFast(Cosmos3OmniModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="Cosmos3 does not support batched prompts.") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="Cosmos3 does not support batched prompts.") - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason="Cosmos3 does not support multiple videos per prompt.") def test_num_images_per_prompt(self): pass diff --git a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py index cf3a38a66b7c..c49e8ea2a1e6 100644 --- a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py +++ b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py @@ -73,6 +73,7 @@ class Cosmos3DistilledModularPipelineTesterConfig(BaseModularPipelineTesterConfi pretrained_model_name_or_path = TINY_DISTILLED_REPO expected_latents_shape = (1, 4, 2, 4, 4) params = frozenset(["prompt", "height", "width", "num_frames"]) + # one prompt per call: the text encoder raises on a list batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "output_type"]) output_name = "videos" @@ -96,14 +97,6 @@ def get_dummy_inputs(self, seed=0): class TestCosmos3DistilledModularPipelineFast(Cosmos3DistilledModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="Cosmos3 does not support batched prompts.") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="Cosmos3 does not support batched prompts.") - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason="Cosmos3 does not support multiple videos per prompt.") def test_num_images_per_prompt(self): pass diff --git a/tests/modular_pipelines/echo/testing_utils.py b/tests/modular_pipelines/echo/testing_utils.py index b85d1d1930c3..08611573a741 100644 --- a/tests/modular_pipelines/echo/testing_utils.py +++ b/tests/modular_pipelines/echo/testing_utils.py @@ -39,7 +39,6 @@ class EchoModularPipelineTesterConfig(BaseModularPipelineTesterConfig): "sigmas", ] ) - batch_params = frozenset(["prompt"]) optional_params = frozenset(["num_videos_per_prompt", "latents", "audio_latents", "output_type"]) not_params = frozenset(["negative_prompt", "guidance_scale", "num_inference_steps"]) expected_workflow_blocks = {} diff --git a/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py b/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py index 04b1d56f86db..e3c2f1492600 100644 --- a/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py +++ b/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py @@ -46,7 +46,6 @@ class ErnieImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): params = frozenset(["prompt", "height", "width"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() - batch_params = frozenset(["prompt"]) optional_params = frozenset(["num_inference_steps", "num_images_per_prompt", "latents"]) expected_workflow_blocks = ERNIE_IMAGE_WORKFLOWS diff --git a/tests/modular_pipelines/flux/test_modular_pipeline_flux.py b/tests/modular_pipelines/flux/test_modular_pipeline_flux.py index 0d076477ebbe..9ae7008cf5d8 100644 --- a/tests/modular_pipelines/flux/test_modular_pipeline_flux.py +++ b/tests/modular_pipelines/flux/test_modular_pipeline_flux.py @@ -61,7 +61,6 @@ class FluxModularPipelineTesterConfig(BaseModularPipelineTesterConfig): expected_latents_shape = (1, 1, 8, 8) params = frozenset(["prompt", "height", "width", "guidance_scale"]) - batch_params = frozenset(["prompt"]) expected_workflow_blocks = FLUX_TEXT2IMAGE_WORKFLOWS def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py index f30e1ad0473f..e76b92c91ecf 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py @@ -57,7 +57,6 @@ class Flux2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): params = frozenset(["prompt", "height", "width", "guidance_scale"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() - batch_params = frozenset(["prompt"]) expected_workflow_blocks = FLUX2_TEXT2IMAGE_WORKFLOWS def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py index 8b75cab1ece9..986f2ba646b7 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py @@ -56,7 +56,6 @@ class Flux2KleinModularPipelineTesterConfig(BaseModularPipelineTesterConfig): params = frozenset(["prompt", "height", "width"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() - batch_params = frozenset(["prompt"]) not_params = frozenset(["negative_prompt"]) expected_workflow_blocks = FLUX2_KLEIN_WORKFLOWS diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py index f47f160fd62d..505199c06bd2 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py @@ -56,7 +56,6 @@ class Flux2KleinBaseModularPipelineTesterConfig(BaseModularPipelineTesterConfig) params = frozenset(["prompt", "height", "width"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() - batch_params = frozenset(["prompt"]) expected_workflow_blocks = FLUX2_KLEIN_BASE_WORKFLOWS def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/flux3/test_modular_pipeline_flux3.py b/tests/modular_pipelines/flux3/test_modular_pipeline_flux3.py index 3d4daa665dc7..9846cb8ca4b0 100644 --- a/tests/modular_pipelines/flux3/test_modular_pipeline_flux3.py +++ b/tests/modular_pipelines/flux3/test_modular_pipeline_flux3.py @@ -181,6 +181,7 @@ class Flux3ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): # The patchified grid: `AutoencoderKLFlux2` defines its latent statistics over patchified channels. expected_latents_shape = (1, 16, 16, 16) params = frozenset(["prompt", "height", "width"]) + # one prompt per call: the prompt upsampler raises on a list batch_params = frozenset() expected_workflow_blocks = FLUX3_WORKFLOWS expected_workflow_defaults = FLUX3_WORKFLOW_DEFAULTS @@ -205,14 +206,6 @@ def get_dummy_inputs(self, seed=0): class TestFlux3ModularPipelineFast(Flux3ModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="FLUX.3 takes one prompt per call; batched prompts are not supported.") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="FLUX.3 takes one prompt per call; batched prompts are not supported.") - def test_inference_batch_single_identical(self): - pass - def test_num_images_per_prompt(self): # one prompt per call, several images from it super().test_num_images_per_prompt(batch_sizes=[1]) @@ -377,14 +370,6 @@ class Flux3DistilledModularPipelineTesterConfig(Flux3ModularPipelineTesterConfig class TestFlux3DistilledModularPipelineFast(Flux3DistilledModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="FLUX.3 takes one prompt per call; batched prompts are not supported.") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="FLUX.3 takes one prompt per call; batched prompts are not supported.") - def test_inference_batch_single_identical(self): - pass - def test_num_images_per_prompt(self): super().test_num_images_per_prompt(batch_sizes=[1]) diff --git a/tests/modular_pipelines/helios/test_modular_pipeline_helios.py b/tests/modular_pipelines/helios/test_modular_pipeline_helios.py index f3e7b43335fb..84b2508edd7d 100644 --- a/tests/modular_pipelines/helios/test_modular_pipeline_helios.py +++ b/tests/modular_pipelines/helios/test_modular_pipeline_helios.py @@ -73,7 +73,6 @@ class HeliosModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-helios-modular-pipe" expected_latents_shape = (1, 16, 9, 2, 2) params = frozenset(["prompt", "height", "width", "num_frames"]) - batch_params = frozenset(["prompt"]) optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) output_name = "videos" expected_workflow_blocks = HELIOS_WORKFLOWS @@ -150,7 +149,6 @@ class HeliosPyramidModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-helios-pyramid-modular-pipe" expected_latents_shape = (1, 16, 9, 8, 8) params = frozenset(["prompt", "height", "width", "num_frames"]) - batch_params = frozenset(["prompt"]) optional_params = frozenset(["pyramid_num_inference_steps_list", "num_videos_per_prompt", "latents"]) output_name = "videos" expected_workflow_blocks = HELIOS_PYRAMID_WORKFLOWS diff --git a/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py b/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py index 3fa9368931ad..2d24d1fdc143 100644 --- a/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py +++ b/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py @@ -57,7 +57,6 @@ class HunyuanVideo15ModularPipelineTesterConfig(BaseModularPipelineTesterConfig) params = frozenset(["prompt", "height", "width", "num_frames"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() - batch_params = frozenset(["prompt"]) optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) expected_workflow_blocks = HUNYUANVIDEO15_WORKFLOWS output_name = "videos" diff --git a/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py b/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py index c2c803e9a16b..b12c50535e7a 100644 --- a/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py +++ b/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py @@ -71,7 +71,8 @@ class KandinskyS2VModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = KandinskyS2VAutoBlocks pretrained_model_name_or_path = TINY_REPO params = frozenset(["image", "audio", "prompt", "height", "width", "num_latent_frames"]) - batch_params = frozenset(["prompt"]) + # one prompt per call: batching is not supported + batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "output_type"]) required_params = frozenset(["audio", "image"]) output_name = "videos" @@ -127,14 +128,6 @@ def get_dummy_inputs(self, seed=0): class TestKandinskyS2VModularPipelineFast(KandinskyS2VModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="The pipeline takes one prompt at a time; batching is not supported") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="The pipeline takes one prompt at a time; batching is not supported") - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason="One video per call; there is no num_images_per_prompt") def test_num_images_per_prompt(self): pass @@ -209,8 +202,6 @@ class TestKandinskyS2VModularPipelineMemory(KandinskyS2VModularPipelineTesterCon FAST_SKIPS = { - "test_inference_batch_consistent": "The pipeline takes one prompt at a time; batching is not supported", - "test_inference_batch_single_identical": "The pipeline takes one prompt at a time; batching is not supported", "test_num_images_per_prompt": "One video per call; there is no num_images_per_prompt", } @@ -259,14 +250,6 @@ def get_dummy_inputs(self, seed=0): class TestKandinskyS2VDistilledModularPipelineFast( KandinskyS2VDistilledModularPipelineTesterConfig, ModularPipelineTesterMixin ): - @pytest.mark.skip(reason=FAST_SKIPS["test_inference_batch_consistent"]) - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason=FAST_SKIPS["test_inference_batch_single_identical"]) - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason=FAST_SKIPS["test_num_images_per_prompt"]) def test_num_images_per_prompt(self): pass diff --git a/tests/modular_pipelines/krea2/test_modular_pipeline_krea2.py b/tests/modular_pipelines/krea2/test_modular_pipeline_krea2.py index 0573823eeaba..5d498a83babf 100644 --- a/tests/modular_pipelines/krea2/test_modular_pipeline_krea2.py +++ b/tests/modular_pipelines/krea2/test_modular_pipeline_krea2.py @@ -49,7 +49,6 @@ class Krea2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-krea2-modular-pipe" expected_latents_shape = (1, 4, 1, 8, 8) params = frozenset(["prompt", "height", "width"]) - batch_params = frozenset(["prompt"]) expected_workflow_blocks = KREA2_WORKFLOWS def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/krea2/test_modular_pipeline_krea2_turbo.py b/tests/modular_pipelines/krea2/test_modular_pipeline_krea2_turbo.py index 7dbe1444cf0e..57c94e3a18a3 100644 --- a/tests/modular_pipelines/krea2/test_modular_pipeline_krea2_turbo.py +++ b/tests/modular_pipelines/krea2/test_modular_pipeline_krea2_turbo.py @@ -45,7 +45,6 @@ class Krea2TurboModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-krea2-turbo-modular-pipe" expected_latents_shape = (1, 4, 1, 8, 8) params = frozenset(["prompt", "height", "width"]) - batch_params = frozenset(["prompt"]) expected_workflow_blocks = KREA2_TURBO_WORKFLOWS def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py b/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py index 3f9df2e52f6c..67ee50b5865e 100644 --- a/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py +++ b/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py @@ -59,7 +59,6 @@ class LTXModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "akshan-main/tiny-ltx-modular-pipe" expected_latents_shape = (1, 8, 5, 16, 16) params = frozenset(["prompt", "height", "width", "num_frames"]) - batch_params = frozenset(["prompt"]) optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) expected_workflow_blocks = LTX_WORKFLOWS output_name = "videos" diff --git a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py index 9e0f4ac29573..b7c5315af2cd 100644 --- a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py +++ b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py @@ -104,7 +104,6 @@ class LTX2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_class = LTX2ModularPipeline pipeline_blocks_class = LTX2AutoBlocks pretrained_model_name_or_path = LTX2_REPO_ID - batch_params = frozenset(["prompt"]) optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) expected_workflow_blocks = LTX2_WORKFLOWS output_name = "videos" diff --git a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py index 1f00e125f93e..3efd5025d75c 100644 --- a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py +++ b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py @@ -102,7 +102,6 @@ class LTX25ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_class = LTX25ModularPipeline pipeline_blocks_class = LTX25AutoBlocks pretrained_model_name_or_path = LTX25_REPO_ID - batch_params = frozenset(["prompt"]) optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) expected_workflow_blocks = LTX25_WORKFLOWS output_name = "videos" diff --git a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py index dd7f04f1c7f1..bdfc09e8f683 100644 --- a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py +++ b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py @@ -173,6 +173,7 @@ class MiniMaxH3ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): expected_latents_shape = (1, 4, 37, 2, 2) params = frozenset(["prompt", "image", "last_image", "height", "width", "num_frames"]) # MiniMax-H3 packs one request into one sequence and rejects a list of prompts, so nothing is batched. + # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() # The checkpoint is guidance-distilled and generates a video, so neither `guidance_scale` nor # `num_images_per_prompt` exists. @@ -197,14 +198,6 @@ def get_dummy_inputs(self, seed=0): class TestMiniMaxH3ModularPipelineFast(MiniMaxH3ModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="MiniMax-H3 packs one request into one sequence, so a batch of prompts is not a thing.") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="MiniMax-H3 packs one request into one sequence, so a batch of prompts is not a thing.") - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason="One request generates one video, so there is no `num_images_per_prompt`.") def test_num_images_per_prompt(self): pass @@ -613,6 +606,7 @@ class MiniMaxH3Ref2VAModularPipelineTesterConfig(BaseModularPipelineTesterConfig expected_latents_shape = (1, 4, 37, 2, 2) params = frozenset(["prompt", "references", "height", "width", "num_frames"]) # MiniMax-H3 packs one request into one sequence and rejects a list of prompts, so nothing is batched. + # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() # The checkpoint is guidance-distilled and generates a video, so neither `guidance_scale` nor # `num_images_per_prompt` exists. @@ -656,14 +650,6 @@ def get_dummy_inputs(self, seed=0): class TestMiniMaxH3Ref2VAModularPipelineFast(MiniMaxH3Ref2VAModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="MiniMax-H3 packs one request into one sequence, so a batch of prompts is not a thing.") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="MiniMax-H3 packs one request into one sequence, so a batch of prompts is not a thing.") - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason="One request generates one video, so there is no `num_images_per_prompt`.") def test_num_images_per_prompt(self): pass diff --git a/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py b/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py index d6f368446a13..a1673619d5a7 100644 --- a/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py +++ b/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py @@ -113,6 +113,7 @@ class MiniMaxH3ImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig) expected_latents_shape = (1, 4, 8, 8) params = frozenset(["prompt", "image", "height", "width"]) # One request is one packed sequence, so prompts are not batched. + # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() optional_params = frozenset(["latents", "output_type"]) output_name = "images" @@ -144,14 +145,6 @@ def get_edit_inputs(self, seed=0): class TestMiniMaxH3ImageModularPipelineFast(MiniMaxH3ImageModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="MiniMax-H3-Image packs one request into one sequence, so there is no batch of prompts.") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="MiniMax-H3-Image packs one request into one sequence, so there is no batch of prompts.") - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason="One request generates one image, so there is no `num_images_per_prompt`.") def test_num_images_per_prompt(self): pass diff --git a/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py b/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py index 45096f326be1..45236080f81d 100644 --- a/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py +++ b/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py @@ -42,7 +42,6 @@ "(dtype, offload-induced kernel changes) legitimately flips sampled tokens, so value-equality " "comparisons cannot hold. Shape and determinism-at-fixed-seed are covered by the other tests." ) -_UNBATCHED_SKIP = "MiniMax Music 3 generates a single waveform per call; `prompt` and `lyrics` are single strings." class MiniMaxMusic3ConditionEncoderFastTests(unittest.TestCase): @@ -71,6 +70,7 @@ class MiniMaxMusic3ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-minimax-music3-modular-pipe" params = frozenset(["prompt", "lyrics", "audio_duration"]) # The pipeline generates a single waveform per call; `prompt` and `lyrics` are single strings. + # one music description per call: the encoder raises on a list batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "output_type"]) output_name = "audios" @@ -119,14 +119,6 @@ def test_float16_inference(self): def test_num_images_per_prompt(self): pass - @pytest.mark.skip(reason=_UNBATCHED_SKIP) - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason=_UNBATCHED_SKIP) - def test_inference_batch_single_identical(self): - pass - def test_output_is_stereo_waveform(self): pipe = self.get_pipeline() diff --git a/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py b/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py index 40f998bccb20..5096a354d009 100644 --- a/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py +++ b/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py @@ -158,6 +158,7 @@ class PH3ImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): expected_latents_shape = (1, 4, 1, 2, 2) params = frozenset(["prompt", "image", "height", "width"]) # One request is one packed sequence, so prompts are not batched. + # one prompt per call: the prompt upsampler raises on a list batch_params = frozenset() # No guidance, one image per request, fixed sampling grid: no `guidance_scale`, `num_images_per_prompt` or # `num_inference_steps`. @@ -188,14 +189,6 @@ def get_edit_inputs(self, seed=0): class TestPH3ImageModularPipelineFast(PH3ImageModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="P-H3-Image packs one request into one sequence, so a batch of prompts is not a thing.") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="P-H3-Image packs one request into one sequence, so a batch of prompts is not a thing.") - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason="One request generates one image, so there is no `num_images_per_prompt`.") def test_num_images_per_prompt(self): pass diff --git a/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py b/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py index 9d18a8a95279..d91ebbc0fc28 100644 --- a/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py +++ b/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py @@ -108,6 +108,7 @@ class PAN2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-pan2-modular-pipe" expected_latents_shape = (1, 48, 2, 2, 2) params = frozenset(["prompt", "height", "width", "num_frames"]) + # one prompt per call: the text encoder raises on a list batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) expected_workflow_blocks = PAN2_WORKFLOWS @@ -140,14 +141,6 @@ def get_dummy_inputs(self, seed=0): class TestPAN2ModularPipelineFast(PAN2ModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="PAN 2 takes a single prompt per call, so a batch of prompts is not a thing.") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="PAN 2 takes a single prompt per call, so a batch of prompts is not a thing.") - def test_inference_batch_single_identical(self): - pass - def test_num_videos_per_prompt(self): pipe = self.get_pipeline().to(torch_device) videos = pipe(**self.get_dummy_inputs(), num_videos_per_prompt=2, output=self.output_name) diff --git a/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py b/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py index 815e7e0536ea..3ed50943740a 100644 --- a/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py +++ b/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py @@ -157,6 +157,7 @@ class PikaH3FlexModularPipelineTesterConfig(BaseModularPipelineTesterConfig): expected_latents_shape = (1, 4, 37, 2, 2) params = frozenset(["prompt", "image", "last_image", "height", "width", "num_frames"]) # Pika H3 Flex packs one request into one sequence and rejects a list of prompts, so nothing is batched. + # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() # The adapters are guidance-distilled and run a fixed six-step schedule, so neither `guidance_scale` nor # `num_inference_steps` exists, and one request generates one video. @@ -181,14 +182,6 @@ def get_dummy_inputs(self, seed=0): class TestPikaH3FlexModularPipelineFast(PikaH3FlexModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="Pika H3 Flex packs one request into one sequence, so a batch of prompts is not a thing.") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="Pika H3 Flex packs one request into one sequence, so a batch of prompts is not a thing.") - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason="One request generates one video, so there is no `num_images_per_prompt`.") def test_num_images_per_prompt(self): pass @@ -395,6 +388,7 @@ class PikaH3FlexRef2VAModularPipelineTesterConfig(BaseModularPipelineTesterConfi expected_latents_shape = (1, 4, 37, 2, 2) params = frozenset(["prompt", "references", "height", "width", "num_frames"]) # Pika H3 Flex packs one request into one sequence and rejects a list of prompts, so nothing is batched. + # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() # The adapters are guidance-distilled and run a fixed six-step schedule, so neither `guidance_scale` nor # `num_inference_steps` exists, and one request generates one video. @@ -423,14 +417,6 @@ def get_dummy_inputs(self, seed=0): class TestPikaH3FlexRef2VAModularPipelineFast(PikaH3FlexRef2VAModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="Pika H3 Flex packs one request into one sequence, so a batch of prompts is not a thing.") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="Pika H3 Flex packs one request into one sequence, so a batch of prompts is not a thing.") - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason="One request generates one video, so there is no `num_images_per_prompt`.") def test_num_images_per_prompt(self): pass diff --git a/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py b/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py index d5cd4136c87d..acdd836dd032 100644 --- a/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py +++ b/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py @@ -129,7 +129,6 @@ class QwenImage21TurboModularPipelineTesterConfig(BaseModularPipelineTesterConfi pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage21-turbo-modular" expected_latents_shape = (1, 8, 1, 2, 2) params = frozenset(["prompt", "height", "width", "output_resolution", "attention_kwargs", "image"]) - batch_params = frozenset(["prompt"]) not_params = frozenset(["negative_prompt"]) expected_workflow_blocks = QWENIMAGE21_TURBO_TEXT2IMAGE_WORKFLOWS expected_workflow_defaults = {"text2image": QWENIMAGE21_TURBO_WORKFLOW_DEFAULTS["text2image"]} diff --git a/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py b/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py index e19ddec61263..22458f31520b 100644 --- a/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py +++ b/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py @@ -53,7 +53,6 @@ class StableDiffusion3ModularPipelineTesterConfig(BaseModularPipelineTesterConfi pretrained_model_name_or_path = "AlanPonnachan/tiny-sd3-modular" expected_latents_shape = (1, 4, 32, 32) params = frozenset(["prompt", "height", "width"]) - batch_params = frozenset(["prompt"]) expected_workflow_blocks = SD3_TEXT2IMAGE_WORKFLOWS def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/testing_utils/common.py b/tests/modular_pipelines/testing_utils/common.py index 3139dd1d8b1a..dd7b9bbbefc6 100644 --- a/tests/modular_pipelines/testing_utils/common.py +++ b/tests/modular_pipelines/testing_utils/common.py @@ -50,12 +50,13 @@ class BaseModularPipelineTesterConfig: # Canonical parameters the pipeline cannot run without. `test_pipeline_required_parameters` asserts the blocks mark # them required; a pipeline that does not take a prompt overrides this with its own, e.g. `{"audio", "image"}` required_params = frozenset(["prompt"]) + # Parameters the batch tests pass as lists. A pipeline that does not take a batch of prompts overrides this with + # `frozenset()`; a pipeline that batches an image as well adds it + batch_params = frozenset(["prompt"]) # Parameters the pipeline deliberately does NOT accept — e.g. `negative_prompt` on a # guidance-distilled pipeline. `test_pipeline_call_signature` asserts they are absent, # so accidentally (re)introducing one fails the test. not_params = frozenset() - # this is modular specific: generator needs to be a intermediate input because it's mutable - intermediate_params = frozenset(["generator"]) # Output type for the pipeline (e.g., "images" for image pipelines, "videos" for video pipelines) # Subclasses can override this to change the expected output type output_name = "images" @@ -117,20 +118,6 @@ def params(self) -> frozenset: "See existing pipeline tests for reference." ) - @property - def batch_params(self) -> frozenset: - raise NotImplementedError( - "You need to set the attribute `batch_params` in the child test class. " - "`batch_params` are the parameters required to be batched when passed to the pipeline's " - "`__call__` method. `pipeline_params.py` provides some common sets of parameters such as " - "`TEXT_TO_IMAGE_BATCH_PARAMS`, `IMAGE_VARIATION_BATCH_PARAMS`, etc... If your pipeline's " - "set of batch arguments has minor changes from one of the common sets of batch arguments, " - "do not make modifications to the existing common sets of batch arguments. I.e. a text to " - "image pipeline `negative_prompt` is not batched should set the attribute as " - "`batch_params = TEXT_TO_IMAGE_BATCH_PARAMS - {'negative_prompt'}`. " - "See existing pipeline tests for reference." - ) - @property def expected_workflow_blocks(self) -> dict: raise NotImplementedError( @@ -273,6 +260,8 @@ def test_required_inputs_are_required(self): pipe(**{k: v for k, v in inputs.items() if k != name}, output=self.output_name) def test_inference_batch_consistent(self, batch_sizes=[2], batch_generator=True): + if not self.batch_params: + pytest.skip("the pipeline takes one prompt per call") pipe = self.get_pipeline().to(torch_device) inputs = self.get_dummy_inputs() @@ -312,6 +301,8 @@ def test_inference_batch_single_identical( batch_size=2, expected_max_diff=1e-4, ): + if not self.batch_params: + pytest.skip("the pipeline takes one prompt per call") pipe = self.get_pipeline().to(torch_device) inputs = self.get_dummy_inputs() diff --git a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py index 2f1e0a7de82b..baec7c9de00a 100644 --- a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py +++ b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py @@ -32,7 +32,6 @@ class WanModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-wan-modular-pipe" expected_latents_shape = (1, 16, 3, 2, 2) params = frozenset(["prompt", "height", "width", "num_frames"]) - batch_params = frozenset(["prompt"]) optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) output_name = "videos" diff --git a/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py b/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py index 3b22389ec632..ba18aab16982 100644 --- a/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py +++ b/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py @@ -36,6 +36,7 @@ class Wan22VaceModularPipelineTesterConfig(BaseModularPipelineTesterConfig): expected_latents_shape = (1, 16, 3, 2, 2) params = frozenset(["prompt", "height", "width", "num_frames", "video", "mask", "reference_images"]) + # a list of prompts is not yet supported: `WanVaceAdditionalInputsStep` raises on one batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) output_name = "videos" @@ -70,14 +71,6 @@ def test_inference_with_reference_image(self): assert videos.shape == (1, 9, 3, 16, 16) assert torch.isnan(videos).sum() == 0 - @pytest.mark.skip(reason="Batching is not yet supported with this pipeline") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="Batching is not yet supported with this pipeline") - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason="num_videos_per_prompt") def test_num_images_per_prompt(self): pass diff --git a/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py b/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py index e7906dfb6187..65d65fa57a4b 100644 --- a/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py +++ b/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py @@ -87,6 +87,7 @@ class WanAnimate2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): # Each segment is decoded inside the denoise loop, so no latents are left in the state. params = frozenset(["prompt", "image", "driving_video"]) + # one prompt per call: the text encoder raises on a list batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "height", "width", "output_type"]) output_name = "videos" @@ -119,14 +120,6 @@ class WanAnimate2ModularPipelineFastTesterMixin(ModularPipelineTesterMixin): def test_latents_output_in_canonical_form(self): pass - @pytest.mark.skip(reason="Wan-Animate-2 is unbatched: one character image and driving video per call") - def test_inference_batch_consistent(self): - pass - - @pytest.mark.skip(reason="Wan-Animate-2 is unbatched: one character image and driving video per call") - def test_inference_batch_single_identical(self): - pass - @pytest.mark.skip(reason="Wan-Animate-2 is unbatched: no num_videos_per_prompt") def test_num_images_per_prompt(self): pass diff --git a/tests/modular_pipelines/z_image/test_modular_pipeline_z_image.py b/tests/modular_pipelines/z_image/test_modular_pipeline_z_image.py index 0c58866359d4..d34fea71ca44 100644 --- a/tests/modular_pipelines/z_image/test_modular_pipeline_z_image.py +++ b/tests/modular_pipelines/z_image/test_modular_pipeline_z_image.py @@ -55,7 +55,6 @@ class ZImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-zimage-modular-pipe" expected_latents_shape = (1, 16, 16, 16) params = frozenset(["prompt", "height", "width"]) - batch_params = frozenset(["prompt"]) expected_workflow_blocks = ZIMAGE_WORKFLOWS def get_dummy_inputs(self, seed=0): From a905d881b6aae7ca1b75183558370b0157000354 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 17 Nov 2026 01:33:39 +0100 Subject: [PATCH 22/33] [modular tests] drop `params`; `test_pipeline_not_parameters` keeps the `not_params` check `required_params` and `optional_params` cover the canonical inputs and `expected_workflow_defaults` pins the exact set per workflow, so the subset check added nothing. The LTX2 text2video configs held only `params` and fold into the shared config. Co-Authored-By: Claude Fable 5.1 --- .../anima/test_modular_pipeline_anima.py | 2 -- .../cosmos/test_modular_pipeline_cosmos3.py | 1 - ...test_modular_pipeline_cosmos3_distilled.py | 1 - tests/modular_pipelines/echo/testing_utils.py | 14 --------- .../test_modular_pipeline_ernie_image.py | 1 - .../flux/test_modular_pipeline_flux.py | 3 -- .../flux2/test_modular_pipeline_flux2.py | 2 -- .../test_modular_pipeline_flux2_klein.py | 2 -- .../test_modular_pipeline_flux2_klein_base.py | 2 -- .../flux3/test_modular_pipeline_flux3.py | 1 - .../helios/test_modular_pipeline_helios.py | 2 -- .../test_modular_pipeline_hunyuan_video1_5.py | 1 - .../test_modular_pipeline_kandinsky_s2v.py | 1 - .../krea2/test_modular_pipeline_krea2.py | 1 - .../test_modular_pipeline_krea2_turbo.py | 1 - .../ltx/test_modular_pipeline_ltx.py | 1 - .../ltx2/test_modular_pipeline_ltx2.py | 22 +++---------- .../ltx2/test_modular_pipeline_ltx25.py | 22 +++---------- .../test_modular_pipeline_minimax_h3.py | 2 -- .../test_modular_pipeline_minimax_h3_image.py | 1 - .../test_modular_pipeline_minimax_music3.py | 1 - .../test_modular_pipeline_p_h3_image.py | 1 - .../pan2/test_modular_pipeline_pan2.py | 1 - .../test_modular_pipeline_pika_h3_flex.py | 2 -- .../qwen/test_modular_pipeline_qwenimage.py | 3 -- .../test_modular_pipeline_qwenimage21.py | 3 -- ...test_modular_pipeline_qwenimage21_turbo.py | 1 - ...est_modular_pipeline_stable_diffusion_3.py | 12 ------- ...st_modular_pipeline_stable_diffusion_xl.py | 30 ------------------ .../modular_pipelines/testing_utils/common.py | 31 +++++-------------- .../wan/test_modular_pipeline_wan.py | 1 - .../wan/test_modular_pipeline_wan_vace.py | 1 - .../test_modular_pipeline_wan_animate_2.py | 1 - .../z_image/test_modular_pipeline_z_image.py | 1 - 34 files changed, 18 insertions(+), 154 deletions(-) diff --git a/tests/modular_pipelines/anima/test_modular_pipeline_anima.py b/tests/modular_pipelines/anima/test_modular_pipeline_anima.py index 625dbd552136..681c3b2b13e4 100644 --- a/tests/modular_pipelines/anima/test_modular_pipeline_anima.py +++ b/tests/modular_pipelines/anima/test_modular_pipeline_anima.py @@ -111,7 +111,6 @@ class AnimaModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = AnimaAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-anima-modular-pipe" expected_latents_shape = (1, 4, 1, 8, 8) - params = frozenset(["prompt", "height", "width", "negative_prompt"]) batch_params = frozenset(["prompt", "negative_prompt"]) expected_workflow_blocks = ANIMA_TEXT2IMAGE_WORKFLOWS @@ -257,7 +256,6 @@ class AnimaImg2ImgModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = AnimaAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-anima-modular-pipe" expected_latents_shape = (1, 4, 1, 8, 8) - params = frozenset(["prompt", "image", "strength", "height", "width", "negative_prompt"]) batch_params = frozenset(["prompt", "negative_prompt"]) expected_workflow_blocks = ANIMA_IMG2IMG_WORKFLOWS diff --git a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py index cf7e2543a6eb..0ae6daa3797e 100644 --- a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py +++ b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py @@ -133,7 +133,6 @@ class Cosmos3OmniModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = Cosmos3OmniBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-cosmos3-modular-pipe" expected_latents_shape = (1, 4, 2, 4, 4) - params = frozenset(["prompt", "height", "width", "num_frames", "guidance_scale"]) # one prompt per call: the text encoder raises on a list batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "output_type"]) diff --git a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py index c49e8ea2a1e6..c20228576a8d 100644 --- a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py +++ b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py @@ -72,7 +72,6 @@ class Cosmos3DistilledModularPipelineTesterConfig(BaseModularPipelineTesterConfi pipeline_blocks_class = Cosmos3DistilledBlocks pretrained_model_name_or_path = TINY_DISTILLED_REPO expected_latents_shape = (1, 4, 2, 4, 4) - params = frozenset(["prompt", "height", "width", "num_frames"]) # one prompt per call: the text encoder raises on a list batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "output_type"]) diff --git a/tests/modular_pipelines/echo/testing_utils.py b/tests/modular_pipelines/echo/testing_utils.py index 08611573a741..6268a728934f 100644 --- a/tests/modular_pipelines/echo/testing_utils.py +++ b/tests/modular_pipelines/echo/testing_utils.py @@ -25,20 +25,6 @@ class EchoModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = EchoBlocks pretrained_model_name_or_path = "Echo-Team/tiny-echo-modular-pipe" expected_latents_shape = (1, 4, 3, 16, 16) - params = frozenset( - [ - "prompt", - "image", - "memory_images", - "memory_audio_waveforms", - "height", - "width", - "num_frames", - "frame_rate", - "model_frame_rate", - "sigmas", - ] - ) optional_params = frozenset(["num_videos_per_prompt", "latents", "audio_latents", "output_type"]) not_params = frozenset(["negative_prompt", "guidance_scale", "num_inference_steps"]) expected_workflow_blocks = {} diff --git a/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py b/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py index e3c2f1492600..ee9aa5db55c0 100644 --- a/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py +++ b/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py @@ -43,7 +43,6 @@ class ErnieImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = ErnieImageAutoBlocks pretrained_model_name_or_path = "akshan-main/tiny-ernie-image-modular-pipe" expected_latents_shape = (1, 16, 2, 2) - params = frozenset(["prompt", "height", "width"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() optional_params = frozenset(["num_inference_steps", "num_images_per_prompt", "latents"]) diff --git a/tests/modular_pipelines/flux/test_modular_pipeline_flux.py b/tests/modular_pipelines/flux/test_modular_pipeline_flux.py index 9ae7008cf5d8..37c494ea546e 100644 --- a/tests/modular_pipelines/flux/test_modular_pipeline_flux.py +++ b/tests/modular_pipelines/flux/test_modular_pipeline_flux.py @@ -60,7 +60,6 @@ class FluxModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-flux-modular" expected_latents_shape = (1, 1, 8, 8) - params = frozenset(["prompt", "height", "width", "guidance_scale"]) expected_workflow_blocks = FLUX_TEXT2IMAGE_WORKFLOWS def get_dummy_inputs(self, seed=0): @@ -136,7 +135,6 @@ class FluxImg2ImgModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-flux-modular" expected_latents_shape = (1, 1, 8, 8) - params = frozenset(["prompt", "height", "width", "guidance_scale", "image"]) batch_params = frozenset(["prompt", "image"]) expected_workflow_blocks = FLUX_IMAGE2IMAGE_WORKFLOWS @@ -236,7 +234,6 @@ class FluxKontextModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-flux-kontext-pipe" expected_latents_shape = (1, 1, 8, 8) - params = frozenset(["prompt", "height", "width", "guidance_scale", "image"]) batch_params = frozenset(["prompt", "image"]) expected_workflow_blocks = FLUX_KONTEXT_WORKFLOWS diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py index e76b92c91ecf..a2a05a1ecd3a 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py @@ -54,7 +54,6 @@ class Flux2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = Flux2AutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-modular" expected_latents_shape = (1, 4, 16, 16) - params = frozenset(["prompt", "height", "width", "guidance_scale"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() expected_workflow_blocks = FLUX2_TEXT2IMAGE_WORKFLOWS @@ -116,7 +115,6 @@ class Flux2ImageConditionedModularPipelineTesterConfig(BaseModularPipelineTester pipeline_blocks_class = Flux2AutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-modular" expected_latents_shape = (1, 4, 16, 16) - params = frozenset(["prompt", "height", "width", "guidance_scale", "image"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() batch_params = frozenset(["prompt", "image"]) diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py index 986f2ba646b7..aba1992384bc 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py @@ -53,7 +53,6 @@ class Flux2KleinModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = Flux2KleinAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-modular" expected_latents_shape = (1, 4, 16, 16) - params = frozenset(["prompt", "height", "width"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() not_params = frozenset(["negative_prompt"]) @@ -114,7 +113,6 @@ class Flux2KleinImageConditionedModularPipelineTesterConfig(BaseModularPipelineT pipeline_blocks_class = Flux2KleinAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-modular" expected_latents_shape = (1, 4, 16, 16) - params = frozenset(["prompt", "height", "width", "image"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() batch_params = frozenset(["prompt", "image"]) diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py index 505199c06bd2..20c92b6801ab 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py @@ -53,7 +53,6 @@ class Flux2KleinBaseModularPipelineTesterConfig(BaseModularPipelineTesterConfig) pipeline_blocks_class = Flux2KleinBaseAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-base-modular" expected_latents_shape = (1, 4, 16, 16) - params = frozenset(["prompt", "height", "width"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() expected_workflow_blocks = FLUX2_KLEIN_BASE_WORKFLOWS @@ -113,7 +112,6 @@ class Flux2KleinBaseImageConditionedModularPipelineTesterConfig(BaseModularPipel pipeline_blocks_class = Flux2KleinBaseAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-base-modular" expected_latents_shape = (1, 4, 16, 16) - params = frozenset(["prompt", "height", "width", "image"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() batch_params = frozenset(["prompt", "image"]) diff --git a/tests/modular_pipelines/flux3/test_modular_pipeline_flux3.py b/tests/modular_pipelines/flux3/test_modular_pipeline_flux3.py index 9846cb8ca4b0..5a8ff900c77e 100644 --- a/tests/modular_pipelines/flux3/test_modular_pipeline_flux3.py +++ b/tests/modular_pipelines/flux3/test_modular_pipeline_flux3.py @@ -180,7 +180,6 @@ class Flux3ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = FLUX3_TINY_REPO # The patchified grid: `AutoencoderKLFlux2` defines its latent statistics over patchified channels. expected_latents_shape = (1, 16, 16, 16) - params = frozenset(["prompt", "height", "width"]) # one prompt per call: the prompt upsampler raises on a list batch_params = frozenset() expected_workflow_blocks = FLUX3_WORKFLOWS diff --git a/tests/modular_pipelines/helios/test_modular_pipeline_helios.py b/tests/modular_pipelines/helios/test_modular_pipeline_helios.py index 84b2508edd7d..c178e0393354 100644 --- a/tests/modular_pipelines/helios/test_modular_pipeline_helios.py +++ b/tests/modular_pipelines/helios/test_modular_pipeline_helios.py @@ -72,7 +72,6 @@ class HeliosModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = HeliosAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-helios-modular-pipe" expected_latents_shape = (1, 16, 9, 2, 2) - params = frozenset(["prompt", "height", "width", "num_frames"]) optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) output_name = "videos" expected_workflow_blocks = HELIOS_WORKFLOWS @@ -148,7 +147,6 @@ class HeliosPyramidModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = HeliosPyramidAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-helios-pyramid-modular-pipe" expected_latents_shape = (1, 16, 9, 8, 8) - params = frozenset(["prompt", "height", "width", "num_frames"]) optional_params = frozenset(["pyramid_num_inference_steps_list", "num_videos_per_prompt", "latents"]) output_name = "videos" expected_workflow_blocks = HELIOS_PYRAMID_WORKFLOWS diff --git a/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py b/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py index 2d24d1fdc143..c8ed94e33298 100644 --- a/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py +++ b/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py @@ -54,7 +54,6 @@ class HunyuanVideo15ModularPipelineTesterConfig(BaseModularPipelineTesterConfig) pipeline_blocks_class = HunyuanVideo15AutoBlocks pretrained_model_name_or_path = "akshan-main/tiny-hunyuanvideo1_5-modular-pipe" expected_latents_shape = (1, 32, 3, 2, 2) - params = frozenset(["prompt", "height", "width", "num_frames"]) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) diff --git a/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py b/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py index b12c50535e7a..88e023a9d02b 100644 --- a/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py +++ b/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py @@ -70,7 +70,6 @@ class KandinskyS2VModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_class = KandinskyS2VModularPipeline pipeline_blocks_class = KandinskyS2VAutoBlocks pretrained_model_name_or_path = TINY_REPO - params = frozenset(["image", "audio", "prompt", "height", "width", "num_latent_frames"]) # one prompt per call: batching is not supported batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "output_type"]) diff --git a/tests/modular_pipelines/krea2/test_modular_pipeline_krea2.py b/tests/modular_pipelines/krea2/test_modular_pipeline_krea2.py index 5d498a83babf..3e0053373d12 100644 --- a/tests/modular_pipelines/krea2/test_modular_pipeline_krea2.py +++ b/tests/modular_pipelines/krea2/test_modular_pipeline_krea2.py @@ -48,7 +48,6 @@ class Krea2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = Krea2AutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-krea2-modular-pipe" expected_latents_shape = (1, 4, 1, 8, 8) - params = frozenset(["prompt", "height", "width"]) expected_workflow_blocks = KREA2_WORKFLOWS def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/krea2/test_modular_pipeline_krea2_turbo.py b/tests/modular_pipelines/krea2/test_modular_pipeline_krea2_turbo.py index 57c94e3a18a3..d1606f485c88 100644 --- a/tests/modular_pipelines/krea2/test_modular_pipeline_krea2_turbo.py +++ b/tests/modular_pipelines/krea2/test_modular_pipeline_krea2_turbo.py @@ -44,7 +44,6 @@ class Krea2TurboModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = Krea2TurboAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-krea2-turbo-modular-pipe" expected_latents_shape = (1, 4, 1, 8, 8) - params = frozenset(["prompt", "height", "width"]) expected_workflow_blocks = KREA2_TURBO_WORKFLOWS def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py b/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py index 67ee50b5865e..1a6fc37da42b 100644 --- a/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py +++ b/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py @@ -58,7 +58,6 @@ class LTXModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = LTXAutoBlocks pretrained_model_name_or_path = "akshan-main/tiny-ltx-modular-pipe" expected_latents_shape = (1, 8, 5, 16, 16) - params = frozenset(["prompt", "height", "width", "num_frames"]) optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) expected_workflow_blocks = LTX_WORKFLOWS output_name = "videos" diff --git a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py index b7c5315af2cd..245cb7ff572f 100644 --- a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py +++ b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py @@ -99,7 +99,7 @@ class LTX2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): - """Shared configuration for every LTX2 workflow; a variant config adds its own `params` and dummy inputs.""" + """Shared configuration for every LTX2 workflow; a variant config adds its own dummy inputs.""" pipeline_class = LTX2ModularPipeline pipeline_blocks_class = LTX2AutoBlocks @@ -140,13 +140,7 @@ def test_inference_batch_single_identical(self): super().test_inference_batch_single_identical(expected_max_diff=1e-3) -class LTX2Text2VideoModularPipelineTesterConfig(LTX2ModularPipelineTesterConfig): - params = frozenset(["prompt", "height", "width", "num_frames"]) - - -class TestLTX2Text2VideoModularPipelineFast( - LTX2Text2VideoModularPipelineTesterConfig, LTX2ModularPipelineFastTesterMixin -): +class TestLTX2Text2VideoModularPipelineFast(LTX2ModularPipelineTesterConfig, LTX2ModularPipelineFastTesterMixin): def test_audio_output(self): pipe = self.get_pipeline().to("cpu") @@ -173,7 +167,7 @@ def test_auto_duration_predicts_a_grid_valid_frame_count(self): assert 0 < num_frames <= round(2.0 * inputs["frame_rate"]) -class TestLTX2Text2VideoModularPipelineLoading(LTX2Text2VideoModularPipelineTesterConfig, ModularLoadingTesterMixin): +class TestLTX2Text2VideoModularPipelineLoading(LTX2ModularPipelineTesterConfig, ModularLoadingTesterMixin): def test_guiders_round_trip_as_pretrained_components(self, tmp_path): # `guider` and `audio_guider` default to `from_config`, so a checkpoint that ships non-default guidance # scales has to declare them as `from_pretrained` components instead. That path records the class as a @@ -208,18 +202,16 @@ def test_guiders_round_trip_as_pretrained_components(self, tmp_path): assert loaded_pipe.audio_guider.config.guidance_scale == 8.0 -class TestLTX2Text2VideoModularPipelineMemory(LTX2Text2VideoModularPipelineTesterConfig, ModularMemoryTesterMixin): +class TestLTX2Text2VideoModularPipelineMemory(LTX2ModularPipelineTesterConfig, ModularMemoryTesterMixin): pass # The four workflows share `LTX2AutoBlocks` and the same repo, so one workflow test class covers all of them. -class TestLTX2ModularPipelineWorkflow(LTX2Text2VideoModularPipelineTesterConfig, ModularWorkflowTesterMixin): +class TestLTX2ModularPipelineWorkflow(LTX2ModularPipelineTesterConfig, ModularWorkflowTesterMixin): pass class LTX2Image2VideoModularPipelineTesterConfig(LTX2ModularPipelineTesterConfig): - params = frozenset(["prompt", "image", "height", "width", "num_frames"]) - def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) rng = np.random.default_rng(seed) @@ -245,8 +237,6 @@ class TestLTX2Image2VideoModularPipelineMemory(LTX2Image2VideoModularPipelineTes class LTX2ConditionModularPipelineTesterConfig(LTX2ModularPipelineTesterConfig): - params = frozenset(["prompt", "conditions", "height", "width", "num_frames"]) - def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) image = torch.rand((1, 3, 32, 32), generator=torch.Generator("cpu").manual_seed(seed)) @@ -270,8 +260,6 @@ class TestLTX2ConditionModularPipelineMemory(LTX2ConditionModularPipelineTesterC class LTX2InContextModularPipelineTesterConfig(LTX2ModularPipelineTesterConfig): - params = frozenset(["prompt", "reference_conditions", "height", "width", "num_frames"]) - def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) video = torch.rand((1, 5, 3, 32, 32), generator=torch.Generator("cpu").manual_seed(seed)) diff --git a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py index 3efd5025d75c..b1e363458a46 100644 --- a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py +++ b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py @@ -97,7 +97,7 @@ class LTX25ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): - """Shared configuration for every LTX-2.5 workflow; a variant config adds its own `params` and dummy inputs.""" + """Shared configuration for every LTX-2.5 workflow; a variant config adds its own dummy inputs.""" pipeline_class = LTX25ModularPipeline pipeline_blocks_class = LTX25AutoBlocks @@ -138,13 +138,7 @@ def test_inference_batch_single_identical(self): super().test_inference_batch_single_identical(expected_max_diff=1e-3) -class LTX25Text2VideoModularPipelineTesterConfig(LTX25ModularPipelineTesterConfig): - params = frozenset(["prompt", "height", "width", "num_frames"]) - - -class TestLTX25Text2VideoModularPipelineFast( - LTX25Text2VideoModularPipelineTesterConfig, LTX25ModularPipelineFastTesterMixin -): +class TestLTX25Text2VideoModularPipelineFast(LTX25ModularPipelineTesterConfig, LTX25ModularPipelineFastTesterMixin): def test_diffusion_decoder_output(self): pipe = self.get_pipeline().to("cpu") @@ -211,22 +205,20 @@ def test_motion_speed_must_be_positive(self): pipe(**self.get_dummy_inputs(), motion_speed=0.0, output="videos") -class TestLTX25Text2VideoModularPipelineLoading(LTX25Text2VideoModularPipelineTesterConfig, ModularLoadingTesterMixin): +class TestLTX25Text2VideoModularPipelineLoading(LTX25ModularPipelineTesterConfig, ModularLoadingTesterMixin): pass -class TestLTX25Text2VideoModularPipelineMemory(LTX25Text2VideoModularPipelineTesterConfig, ModularMemoryTesterMixin): +class TestLTX25Text2VideoModularPipelineMemory(LTX25ModularPipelineTesterConfig, ModularMemoryTesterMixin): pass # The four workflows share `LTX25AutoBlocks` and the same repo, so one workflow test class covers all of them. -class TestLTX25ModularPipelineWorkflow(LTX25Text2VideoModularPipelineTesterConfig, ModularWorkflowTesterMixin): +class TestLTX25ModularPipelineWorkflow(LTX25ModularPipelineTesterConfig, ModularWorkflowTesterMixin): pass class LTX25Image2VideoModularPipelineTesterConfig(LTX25ModularPipelineTesterConfig): - params = frozenset(["prompt", "image", "height", "width", "num_frames"]) - def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) rng = np.random.default_rng(seed) @@ -254,8 +246,6 @@ class TestLTX25Image2VideoModularPipelineMemory(LTX25Image2VideoModularPipelineT class LTX25ConditionModularPipelineTesterConfig(LTX25ModularPipelineTesterConfig): - params = frozenset(["prompt", "conditions", "height", "width", "num_frames"]) - def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) image = torch.rand((1, 3, 32, 32), generator=torch.Generator("cpu").manual_seed(seed)) @@ -279,8 +269,6 @@ class TestLTX25ConditionModularPipelineMemory(LTX25ConditionModularPipelineTeste class LTX25InContextModularPipelineTesterConfig(LTX25ModularPipelineTesterConfig): - params = frozenset(["prompt", "reference_conditions", "height", "width", "num_frames"]) - def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) video = torch.rand((1, 5, 3, 32, 32), generator=torch.Generator("cpu").manual_seed(seed)) diff --git a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py index bdfc09e8f683..56da51e86cc4 100644 --- a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py +++ b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py @@ -171,7 +171,6 @@ class MiniMaxH3ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = MiniMaxH3Blocks pretrained_model_name_or_path = "hf-internal-testing/tiny-minimax-h3-modular-pipe" expected_latents_shape = (1, 4, 37, 2, 2) - params = frozenset(["prompt", "image", "last_image", "height", "width", "num_frames"]) # MiniMax-H3 packs one request into one sequence and rejects a list of prompts, so nothing is batched. # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() @@ -604,7 +603,6 @@ class MiniMaxH3Ref2VAModularPipelineTesterConfig(BaseModularPipelineTesterConfig pipeline_blocks_class = MiniMaxH3Blocks pretrained_model_name_or_path = "hf-internal-testing/tiny-minimax-h3-modular-pipe" expected_latents_shape = (1, 4, 37, 2, 2) - params = frozenset(["prompt", "references", "height", "width", "num_frames"]) # MiniMax-H3 packs one request into one sequence and rejects a list of prompts, so nothing is batched. # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() diff --git a/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py b/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py index a1673619d5a7..7afce9186179 100644 --- a/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py +++ b/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py @@ -111,7 +111,6 @@ class MiniMaxH3ImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig) pretrained_model_name_or_path = "hf-internal-testing/tiny-minimax-h3-image-modular-pipe" # 4 latent channels at a quarter of a 32 x 32 image: the tiny VAE compresses 4x. expected_latents_shape = (1, 4, 8, 8) - params = frozenset(["prompt", "image", "height", "width"]) # One request is one packed sequence, so prompts are not batched. # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() diff --git a/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py b/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py index 45236080f81d..85a29b445f26 100644 --- a/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py +++ b/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py @@ -68,7 +68,6 @@ class MiniMaxMusic3ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_class = MiniMaxMusic3ModularPipeline pipeline_blocks_class = MiniMaxMusic3Blocks pretrained_model_name_or_path = "hf-internal-testing/tiny-minimax-music3-modular-pipe" - params = frozenset(["prompt", "lyrics", "audio_duration"]) # The pipeline generates a single waveform per call; `prompt` and `lyrics` are single strings. # one music description per call: the encoder raises on a list batch_params = frozenset() diff --git a/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py b/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py index 5096a354d009..f667b3fa5e7b 100644 --- a/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py +++ b/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py @@ -156,7 +156,6 @@ class PH3ImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-p-h3-image-modular-pipe" # One 4-channel latent frame for a 32 x 32 image. expected_latents_shape = (1, 4, 1, 2, 2) - params = frozenset(["prompt", "image", "height", "width"]) # One request is one packed sequence, so prompts are not batched. # one prompt per call: the prompt upsampler raises on a list batch_params = frozenset() diff --git a/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py b/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py index d91ebbc0fc28..b180d6228d01 100644 --- a/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py +++ b/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py @@ -107,7 +107,6 @@ class PAN2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = PAN2Blocks pretrained_model_name_or_path = "hf-internal-testing/tiny-pan2-modular-pipe" expected_latents_shape = (1, 48, 2, 2, 2) - params = frozenset(["prompt", "height", "width", "num_frames"]) # one prompt per call: the text encoder raises on a list batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) diff --git a/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py b/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py index 3ed50943740a..7e4776a9d43a 100644 --- a/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py +++ b/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py @@ -155,7 +155,6 @@ class PikaH3FlexModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = PikaH3FlexBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-pika-h3-flex-modular-pipe" expected_latents_shape = (1, 4, 37, 2, 2) - params = frozenset(["prompt", "image", "last_image", "height", "width", "num_frames"]) # Pika H3 Flex packs one request into one sequence and rejects a list of prompts, so nothing is batched. # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() @@ -386,7 +385,6 @@ class PikaH3FlexRef2VAModularPipelineTesterConfig(BaseModularPipelineTesterConfi pipeline_blocks_class = PikaH3FlexBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-pika-h3-flex-modular-pipe" expected_latents_shape = (1, 4, 37, 2, 2) - params = frozenset(["prompt", "references", "height", "width", "num_frames"]) # Pika H3 Flex packs one request into one sequence and rejects a list of prompts, so nothing is batched. # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() diff --git a/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py b/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py index 85aeb06006d1..e7fe9f5b0033 100644 --- a/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py +++ b/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py @@ -139,7 +139,6 @@ class QwenImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = QwenImageAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage-modular" expected_latents_shape = (1, 4, 1, 8, 8) - params = frozenset(["prompt", "height", "width", "negative_prompt", "attention_kwargs", "image", "mask_image"]) batch_params = frozenset(["prompt", "negative_prompt", "image", "mask_image"]) expected_workflow_blocks = QWEN_IMAGE_TEXT2IMAGE_WORKFLOWS @@ -222,7 +221,6 @@ class QwenImageEditModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = QwenImageEditAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage-edit-modular" expected_latents_shape = (1, 4, 1, 8, 8) - params = frozenset(["prompt", "height", "width", "negative_prompt", "attention_kwargs", "image", "mask_image"]) batch_params = frozenset(["prompt", "negative_prompt", "image", "mask_image"]) expected_workflow_blocks = QWEN_IMAGE_EDIT_WORKFLOWS @@ -272,7 +270,6 @@ class QwenImageEditPlusModularPipelineTesterConfig(BaseModularPipelineTesterConf pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage-edit-plus-modular" expected_latents_shape = (1, 4, 1, 8, 8) # No `mask_image` yet. - params = frozenset(["prompt", "height", "width", "negative_prompt", "attention_kwargs", "image"]) batch_params = frozenset(["prompt", "negative_prompt", "image"]) def get_dummy_inputs(self): diff --git a/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21.py b/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21.py index 5a15c3effb9b..4f98cacf06d5 100644 --- a/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21.py +++ b/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21.py @@ -135,9 +135,6 @@ class QwenImage21ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = QwenImage21AutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage21-modular" expected_latents_shape = (1, 8, 1, 2, 2) - params = frozenset( - ["prompt", "negative_prompt", "height", "width", "output_resolution", "attention_kwargs", "image"] - ) batch_params = frozenset(["prompt", "negative_prompt"]) expected_workflow_blocks = QWENIMAGE21_TEXT2IMAGE_WORKFLOWS expected_workflow_defaults = {"text2image": QWENIMAGE21_WORKFLOW_DEFAULTS["text2image"]} diff --git a/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py b/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py index acdd836dd032..6f2c3638a828 100644 --- a/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py +++ b/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py @@ -128,7 +128,6 @@ class QwenImage21TurboModularPipelineTesterConfig(BaseModularPipelineTesterConfi pipeline_blocks_class = QwenImage21TurboAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage21-turbo-modular" expected_latents_shape = (1, 8, 1, 2, 2) - params = frozenset(["prompt", "height", "width", "output_resolution", "attention_kwargs", "image"]) not_params = frozenset(["negative_prompt"]) expected_workflow_blocks = QWENIMAGE21_TURBO_TEXT2IMAGE_WORKFLOWS expected_workflow_defaults = {"text2image": QWENIMAGE21_TURBO_WORKFLOW_DEFAULTS["text2image"]} diff --git a/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py b/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py index 22458f31520b..26bfbde5539e 100644 --- a/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py +++ b/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py @@ -52,7 +52,6 @@ class StableDiffusion3ModularPipelineTesterConfig(BaseModularPipelineTesterConfi pipeline_blocks_class = StableDiffusion3AutoBlocks pretrained_model_name_or_path = "AlanPonnachan/tiny-sd3-modular" expected_latents_shape = (1, 4, 32, 32) - params = frozenset(["prompt", "height", "width"]) expected_workflow_blocks = SD3_TEXT2IMAGE_WORKFLOWS def get_dummy_inputs(self, seed=0): @@ -69,11 +68,6 @@ def get_dummy_inputs(self, seed=0): class TestStableDiffusion3ModularPipelineFast(StableDiffusion3ModularPipelineTesterConfig, ModularPipelineTesterMixin): - def test_pipeline_call_signature(self): - # Override to prevent signature check failure for guider configurations - # (guidance_scale) which are intentionally omitted from pipeline inputs. - pass - def test_float16_inference(self): super().test_float16_inference(9e-2) @@ -144,7 +138,6 @@ class StableDiffusion3Img2ImgModularPipelineTesterConfig(BaseModularPipelineTest pipeline_blocks_class = StableDiffusion3AutoBlocks pretrained_model_name_or_path = "AlanPonnachan/tiny-sd3-modular" expected_latents_shape = (1, 4, 32, 32) - params = frozenset(["prompt", "height", "width", "image"]) batch_params = frozenset(["prompt", "image"]) expected_workflow_blocks = SD3_IMAGE2IMAGE_WORKFLOWS @@ -175,11 +168,6 @@ def get_dummy_inputs(self, seed=0): class TestStableDiffusion3Img2ImgModularPipelineFast( StableDiffusion3Img2ImgModularPipelineTesterConfig, ModularPipelineTesterMixin ): - def test_pipeline_call_signature(self): - # Override to prevent signature check failure for guider configurations - # (guidance_scale) which are intentionally omitted from pipeline inputs. - pass - def test_float16_inference(self): super().test_float16_inference(9e-2) diff --git a/tests/modular_pipelines/stable_diffusion_xl/test_modular_pipeline_stable_diffusion_xl.py b/tests/modular_pipelines/stable_diffusion_xl/test_modular_pipeline_stable_diffusion_xl.py index b540f6306bfd..64725f9124f9 100644 --- a/tests/modular_pipelines/stable_diffusion_xl/test_modular_pipeline_stable_diffusion_xl.py +++ b/tests/modular_pipelines/stable_diffusion_xl/test_modular_pipeline_stable_diffusion_xl.py @@ -337,15 +337,6 @@ class SDXLModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = StableDiffusionXLAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-sdxl-modular" expected_latents_shape = (1, 4, 32, 32) - params = frozenset( - [ - "prompt", - "height", - "width", - "negative_prompt", - "cross_attention_kwargs", - ] - ) batch_params = frozenset(["prompt", "negative_prompt"]) expected_image_output_shape = (1, 3, 64, 64) @@ -466,16 +457,6 @@ class SDXLImg2ImgModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = StableDiffusionXLAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-sdxl-modular" expected_latents_shape = (1, 4, 32, 32) - params = frozenset( - [ - "prompt", - "height", - "width", - "negative_prompt", - "cross_attention_kwargs", - "image", - ] - ) batch_params = frozenset(["prompt", "negative_prompt", "image"]) expected_image_output_shape = (1, 3, 64, 64) expected_workflow_blocks = IMAGE2IMAGE_WORKFLOWS @@ -611,17 +592,6 @@ class SDXLInpaintingModularPipelineFastTests( pipeline_blocks_class = StableDiffusionXLAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-sdxl-modular" expected_latents_shape = (1, 4, 32, 32) - params = frozenset( - [ - "prompt", - "height", - "width", - "negative_prompt", - "cross_attention_kwargs", - "image", - "mask_image", - ] - ) batch_params = frozenset(["prompt", "negative_prompt", "image", "mask_image"]) expected_image_output_shape = (1, 3, 64, 64) expected_workflow_blocks = INPAINTING_WORKFLOWS diff --git a/tests/modular_pipelines/testing_utils/common.py b/tests/modular_pipelines/testing_utils/common.py index dd7b9bbbefc6..0bc62861a826 100644 --- a/tests/modular_pipelines/testing_utils/common.py +++ b/tests/modular_pipelines/testing_utils/common.py @@ -37,9 +37,10 @@ class BaseModularPipelineTesterConfig: Base class defining the configuration interface for modular pipeline testing. A concrete config must set `pipeline_blocks_class` and `pretrained_model_name_or_path` and implement - `get_dummy_inputs()`; `params` and `batch_params` declare which inputs the blocks are expected to accept and which - of them are batched. This class only declares the testing contract; the pipeline builder and the cached reference - output live on `BaseModularPipelineOutputMixin` (mirroring the non-modular `BasePipelineTesterConfig`). + `get_dummy_inputs()`; `required_params`, `optional_params`, `not_params` and `batch_params` declare what the blocks + are expected to require, accept, reject and batch. This class only declares the testing contract; the pipeline + builder and the cached reference output live on `BaseModularPipelineOutputMixin` (mirroring the non-modular + `BasePipelineTesterConfig`). """ # Canonical parameters that are passed to `__call__` regardless of the type of pipeline. They are always @@ -54,7 +55,7 @@ class BaseModularPipelineTesterConfig: # `frozenset()`; a pipeline that batches an image as well adds it batch_params = frozenset(["prompt"]) # Parameters the pipeline deliberately does NOT accept — e.g. `negative_prompt` on a - # guidance-distilled pipeline. `test_pipeline_call_signature` asserts they are absent, + # guidance-distilled pipeline. `test_pipeline_not_parameters` asserts they are absent, # so accidentally (re)introducing one fails the test. not_params = frozenset() # Output type for the pipeline (e.g., "images" for image pipelines, "videos" for video pipelines) @@ -103,21 +104,6 @@ def get_dummy_inputs(self, seed=0): "See existing pipeline tests for reference." ) - @property - def params(self) -> frozenset: - raise NotImplementedError( - "You need to set the attribute `params` in the child test class. " - "`params` are checked for if all values are present in `__call__`'s signature." - " You can set `params` using one of the common set of parameters defined in `pipeline_params.py`" - " e.g., `TEXT_TO_IMAGE_PARAMS` defines the common parameters used in text to " - "image pipelines, including prompts and prompt embedding overrides." - "If your pipeline's set of arguments has minor changes from one of the common sets of arguments, " - "do not make modifications to the existing common sets of arguments. I.e. a text to image pipeline " - "with non-configurable height and width arguments should set the attribute as " - "`params = TEXT_TO_IMAGE_PARAMS - {'height', 'width'}`. " - "See existing pipeline tests for reference." - ) - @property def expected_workflow_blocks(self) -> dict: raise NotImplementedError( @@ -197,12 +183,11 @@ class ModularPipelineTesterMixin(BaseModularPipelineOutputMixin): `pretrained_model_name_or_path`, `get_dummy_inputs()` and the shared fixtures). """ - def test_pipeline_call_signature(self): - # test all the `params` are present in the pipeline inputs and `not_params` are not present + def test_pipeline_not_parameters(self): + # test to make sure none of `self.not_params` is a pipeline input, e.g. `negative_prompt` on a distilled + # pipeline blocks = self.pipeline_blocks_class() input_parameters = set(blocks.input_names) - assert self.params <= input_parameters, self.params - input_parameters - unsupported_parameters = {param for param in self.not_params if param in input_parameters} assert len(unsupported_parameters) == 0, ( f"Parameters declared in `not_params` unexpectedly present in the pipeline inputs: {unsupported_parameters}" diff --git a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py index baec7c9de00a..f95e61e361f9 100644 --- a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py +++ b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py @@ -31,7 +31,6 @@ class WanModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = WanBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-wan-modular-pipe" expected_latents_shape = (1, 16, 3, 2, 2) - params = frozenset(["prompt", "height", "width", "num_frames"]) optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) output_name = "videos" diff --git a/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py b/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py index ba18aab16982..9ac353510d9b 100644 --- a/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py +++ b/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py @@ -35,7 +35,6 @@ class Wan22VaceModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "akshan-main/tiny-wan22-vace-modular-pipe" expected_latents_shape = (1, 16, 3, 2, 2) - params = frozenset(["prompt", "height", "width", "num_frames", "video", "mask", "reference_images"]) # a list of prompts is not yet supported: `WanVaceAdditionalInputsStep` raises on one batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) diff --git a/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py b/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py index 65d65fa57a4b..21bf2f046448 100644 --- a/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py +++ b/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py @@ -86,7 +86,6 @@ class WanAnimate2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-wan-animate-2-modular" # Each segment is decoded inside the denoise loop, so no latents are left in the state. - params = frozenset(["prompt", "image", "driving_video"]) # one prompt per call: the text encoder raises on a list batch_params = frozenset() optional_params = frozenset(["num_inference_steps", "height", "width", "output_type"]) diff --git a/tests/modular_pipelines/z_image/test_modular_pipeline_z_image.py b/tests/modular_pipelines/z_image/test_modular_pipeline_z_image.py index d34fea71ca44..5d2c4d537f28 100644 --- a/tests/modular_pipelines/z_image/test_modular_pipeline_z_image.py +++ b/tests/modular_pipelines/z_image/test_modular_pipeline_z_image.py @@ -54,7 +54,6 @@ class ZImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = ZImageAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-zimage-modular-pipe" expected_latents_shape = (1, 16, 16, 16) - params = frozenset(["prompt", "height", "width"]) expected_workflow_blocks = ZIMAGE_WORKFLOWS def get_dummy_inputs(self, seed=0): From ed8796737db896ff5a8b00006d6c40492bd1eb80 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 17 Nov 2026 01:45:45 +0100 Subject: [PATCH 23/33] [modular tests] `optional_params` overrides compose from `OPTIONAL_PARAMS` Each override now says what the pipeline lacks or renames instead of restating the set, so the defaults reach every pipeline. The sets were rederived from the pipelines' optional inputs. Co-Authored-By: Claude Fable 5.1 --- .../cosmos/test_modular_pipeline_cosmos3.py | 3 ++- .../test_modular_pipeline_cosmos3_distilled.py | 3 ++- tests/modular_pipelines/echo/testing_utils.py | 7 +++++-- .../test_modular_pipeline_ernie_image.py | 1 - .../helios/test_modular_pipeline_helios.py | 8 ++++++-- .../test_modular_pipeline_hunyuan_video1_5.py | 3 ++- .../test_modular_pipeline_kandinsky_s2v.py | 3 ++- .../ltx/test_modular_pipeline_ltx.py | 3 ++- .../ltx2/test_modular_pipeline_ltx2.py | 3 ++- .../ltx2/test_modular_pipeline_ltx25.py | 3 ++- .../minimax_h3/test_modular_pipeline_minimax_h3.py | 5 +++-- .../test_modular_pipeline_minimax_h3_image.py | 3 ++- .../test_modular_pipeline_minimax_music3.py | 3 ++- .../p_h3_image/test_modular_pipeline_p_h3_image.py | 3 ++- .../pan2/test_modular_pipeline_pan2.py | 3 ++- .../test_modular_pipeline_pika_h3_flex.py | 5 +++-- tests/modular_pipelines/testing_utils/__init__.py | 2 ++ tests/modular_pipelines/testing_utils/common.py | 13 +++++++++---- .../wan/test_modular_pipeline_wan.py | 3 ++- .../wan/test_modular_pipeline_wan_vace.py | 3 ++- .../test_modular_pipeline_wan_animate_2.py | 3 ++- 21 files changed, 56 insertions(+), 27 deletions(-) diff --git a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py index 0ae6daa3797e..eeed6f133251 100644 --- a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py +++ b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py @@ -43,6 +43,7 @@ from ...testing_utils import torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -135,7 +136,7 @@ class Cosmos3OmniModularPipelineTesterConfig(BaseModularPipelineTesterConfig): expected_latents_shape = (1, 4, 2, 4, 4) # one prompt per call: the text encoder raises on a list batch_params = frozenset() - optional_params = frozenset(["num_inference_steps", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} output_name = "videos" expected_workflow_blocks = COSMOS3_OMNI_WORKFLOWS diff --git a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py index c20228576a8d..c38c4cd40fce 100644 --- a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py +++ b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py @@ -24,6 +24,7 @@ from ...testing_utils import torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -74,7 +75,7 @@ class Cosmos3DistilledModularPipelineTesterConfig(BaseModularPipelineTesterConfi expected_latents_shape = (1, 4, 2, 4, 4) # one prompt per call: the text encoder raises on a list batch_params = frozenset() - optional_params = frozenset(["num_inference_steps", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} output_name = "videos" expected_workflow_blocks = COSMOS3_DISTILLED_WORKFLOWS diff --git a/tests/modular_pipelines/echo/testing_utils.py b/tests/modular_pipelines/echo/testing_utils.py index 6268a728934f..85534389dee7 100644 --- a/tests/modular_pipelines/echo/testing_utils.py +++ b/tests/modular_pipelines/echo/testing_utils.py @@ -17,7 +17,7 @@ from diffusers.modular_pipelines import EchoBlocks, EchoModularPipeline -from ..testing_utils import BaseModularPipelineTesterConfig +from ..testing_utils import OPTIONAL_PARAMS, BaseModularPipelineTesterConfig class EchoModularPipelineTesterConfig(BaseModularPipelineTesterConfig): @@ -25,7 +25,10 @@ class EchoModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = EchoBlocks pretrained_model_name_or_path = "Echo-Team/tiny-echo-modular-pipe" expected_latents_shape = (1, 4, 3, 16, 16) - optional_params = frozenset(["num_videos_per_prompt", "latents", "audio_latents", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} | { + "audio_latents", + "num_videos_per_prompt", + } not_params = frozenset(["negative_prompt", "guidance_scale", "num_inference_steps"]) expected_workflow_blocks = {} output_name = "videos" diff --git a/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py b/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py index ee9aa5db55c0..24170eaf310b 100644 --- a/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py +++ b/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py @@ -45,7 +45,6 @@ class ErnieImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): expected_latents_shape = (1, 16, 2, 2) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() - optional_params = frozenset(["num_inference_steps", "num_images_per_prompt", "latents"]) expected_workflow_blocks = ERNIE_IMAGE_WORKFLOWS def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/helios/test_modular_pipeline_helios.py b/tests/modular_pipelines/helios/test_modular_pipeline_helios.py index c178e0393354..5519caea3d5d 100644 --- a/tests/modular_pipelines/helios/test_modular_pipeline_helios.py +++ b/tests/modular_pipelines/helios/test_modular_pipeline_helios.py @@ -23,6 +23,7 @@ ) from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -72,7 +73,7 @@ class HeliosModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = HeliosAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-helios-modular-pipe" expected_latents_shape = (1, 16, 9, 2, 2) - optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} output_name = "videos" expected_workflow_blocks = HELIOS_WORKFLOWS @@ -147,7 +148,10 @@ class HeliosPyramidModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = HeliosPyramidAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-helios-pyramid-modular-pipe" expected_latents_shape = (1, 16, 9, 8, 8) - optional_params = frozenset(["pyramid_num_inference_steps_list", "num_videos_per_prompt", "latents"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} | { + "num_videos_per_prompt", + "pyramid_num_inference_steps_list", + } output_name = "videos" expected_workflow_blocks = HELIOS_PYRAMID_WORKFLOWS diff --git a/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py b/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py index c8ed94e33298..e34e952cec01 100644 --- a/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py +++ b/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py @@ -18,6 +18,7 @@ from diffusers.modular_pipelines import HunyuanVideo15AutoBlocks, HunyuanVideo15ModularPipeline from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -56,7 +57,7 @@ class HunyuanVideo15ModularPipelineTesterConfig(BaseModularPipelineTesterConfig) expected_latents_shape = (1, 32, 3, 2, 2) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() - optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} expected_workflow_blocks = HUNYUANVIDEO15_WORKFLOWS output_name = "videos" diff --git a/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py b/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py index 88e023a9d02b..075ca1c56632 100644 --- a/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py +++ b/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py @@ -30,6 +30,7 @@ from ...testing_utils import numpy_cosine_similarity_distance, require_accelerator, torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularGuiderTesterMixin, ModularLoadingTesterMixin, @@ -72,7 +73,7 @@ class KandinskyS2VModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = TINY_REPO # one prompt per call: batching is not supported batch_params = frozenset() - optional_params = frozenset(["num_inference_steps", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"latents", "num_images_per_prompt"} required_params = frozenset(["audio", "image"]) output_name = "videos" latents_output_name = "latent_chunks" diff --git a/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py b/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py index 1a6fc37da42b..3307aba2a29d 100644 --- a/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py +++ b/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py @@ -21,6 +21,7 @@ from ...testing_utils import torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -58,7 +59,7 @@ class LTXModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = LTXAutoBlocks pretrained_model_name_or_path = "akshan-main/tiny-ltx-modular-pipe" expected_latents_shape = (1, 8, 5, 16, 16) - optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} expected_workflow_blocks = LTX_WORKFLOWS output_name = "videos" diff --git a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py index 245cb7ff572f..d273e3f99510 100644 --- a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py +++ b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py @@ -26,6 +26,7 @@ from diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -104,7 +105,7 @@ class LTX2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_class = LTX2ModularPipeline pipeline_blocks_class = LTX2AutoBlocks pretrained_model_name_or_path = LTX2_REPO_ID - optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} expected_workflow_blocks = LTX2_WORKFLOWS output_name = "videos" diff --git a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py index b1e363458a46..943664599f22 100644 --- a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py +++ b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py @@ -24,6 +24,7 @@ from ...testing_utils import assert_tensors_close from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -102,7 +103,7 @@ class LTX25ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_class = LTX25ModularPipeline pipeline_blocks_class = LTX25AutoBlocks pretrained_model_name_or_path = LTX25_REPO_ID - optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} expected_workflow_blocks = LTX25_WORKFLOWS output_name = "videos" diff --git a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py index 56da51e86cc4..53a0318ba233 100644 --- a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py +++ b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py @@ -42,6 +42,7 @@ from ...testing_utils import CaptureLogger from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularLoraTesterMixin, @@ -176,7 +177,7 @@ class MiniMaxH3ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): batch_params = frozenset() # The checkpoint is guidance-distilled and generates a video, so neither `guidance_scale` nor # `num_images_per_prompt` exists. - optional_params = frozenset(["num_inference_steps", "latents", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} output_name = "videos" expected_workflow_blocks = MINIMAX_H3_WORKFLOWS expected_workflow_defaults = MINIMAX_H3_WORKFLOW_DEFAULTS @@ -608,7 +609,7 @@ class MiniMaxH3Ref2VAModularPipelineTesterConfig(BaseModularPipelineTesterConfig batch_params = frozenset() # The checkpoint is guidance-distilled and generates a video, so neither `guidance_scale` nor # `num_images_per_prompt` exists. - optional_params = frozenset(["num_inference_steps", "latents", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} output_name = "videos" expected_workflow_blocks = MINIMAX_H3_REF2VA_WORKFLOWS expected_workflow_defaults = MINIMAX_H3_REF2VA_WORKFLOW_DEFAULTS diff --git a/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py b/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py index 7afce9186179..0a36ca18a602 100644 --- a/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py +++ b/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py @@ -25,6 +25,7 @@ ) from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -114,7 +115,7 @@ class MiniMaxH3ImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig) # One request is one packed sequence, so prompts are not batched. # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() - optional_params = frozenset(["latents", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} output_name = "images" expected_workflow_blocks = MINIMAX_H3_IMAGE_WORKFLOWS expected_workflow_defaults = MINIMAX_H3_IMAGE_WORKFLOW_DEFAULTS diff --git a/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py b/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py index 85a29b445f26..a11e266985a1 100644 --- a/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py +++ b/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py @@ -26,6 +26,7 @@ from ...testing_utils import enable_full_determinism, torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -71,7 +72,7 @@ class MiniMaxMusic3ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): # The pipeline generates a single waveform per call; `prompt` and `lyrics` are single strings. # one music description per call: the encoder raises on a list batch_params = frozenset() - optional_params = frozenset(["num_inference_steps", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"height", "latents", "num_images_per_prompt", "width"} output_name = "audios" latents_output_name = "latent_chunks" expected_latents_shape = (1, 8, 17) diff --git a/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py b/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py index f667b3fa5e7b..11b2070b3a80 100644 --- a/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py +++ b/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py @@ -35,6 +35,7 @@ from ...testing_utils import CaptureLogger from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -161,7 +162,7 @@ class PH3ImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): batch_params = frozenset() # No guidance, one image per request, fixed sampling grid: no `guidance_scale`, `num_images_per_prompt` or # `num_inference_steps`. - optional_params = frozenset(["latents", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} output_name = "images" expected_workflow_blocks = P_H3_IMAGE_WORKFLOWS expected_workflow_defaults = P_H3_IMAGE_WORKFLOW_DEFAULTS diff --git a/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py b/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py index b180d6228d01..44afdfdf0b05 100644 --- a/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py +++ b/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py @@ -27,6 +27,7 @@ from ...testing_utils import torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularGuiderTesterMixin, ModularLoadingTesterMixin, @@ -109,7 +110,7 @@ class PAN2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): expected_latents_shape = (1, 48, 2, 2, 2) # one prompt per call: the text encoder raises on a list batch_params = frozenset() - optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} expected_workflow_blocks = PAN2_WORKFLOWS expected_workflow_defaults = PAN2_WORKFLOW_DEFAULTS output_name = "videos" diff --git a/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py b/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py index 7e4776a9d43a..9bbc64257648 100644 --- a/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py +++ b/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py @@ -31,6 +31,7 @@ from diffusers.modular_pipelines.pika_h3_flex.modular_pipeline import PIKA_H3_FLEX_FPS, resolve_tier_rule from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularLoraTesterMixin, @@ -160,7 +161,7 @@ class PikaH3FlexModularPipelineTesterConfig(BaseModularPipelineTesterConfig): batch_params = frozenset() # The adapters are guidance-distilled and run a fixed six-step schedule, so neither `guidance_scale` nor # `num_inference_steps` exists, and one request generates one video. - optional_params = frozenset(["latents", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} not_params = frozenset(["num_inference_steps", "guidance_scale", "negative_prompt"]) output_name = "videos" expected_workflow_blocks = PIKA_H3_FLEX_WORKFLOWS @@ -390,7 +391,7 @@ class PikaH3FlexRef2VAModularPipelineTesterConfig(BaseModularPipelineTesterConfi batch_params = frozenset() # The adapters are guidance-distilled and run a fixed six-step schedule, so neither `guidance_scale` nor # `num_inference_steps` exists, and one request generates one video. - optional_params = frozenset(["latents", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} not_params = frozenset(["num_inference_steps", "guidance_scale", "negative_prompt"]) output_name = "videos" expected_workflow_blocks = PIKA_H3_FLEX_REF2VA_WORKFLOWS diff --git a/tests/modular_pipelines/testing_utils/__init__.py b/tests/modular_pipelines/testing_utils/__init__.py index e02c3a1be337..3b05f5e35b3f 100644 --- a/tests/modular_pipelines/testing_utils/__init__.py +++ b/tests/modular_pipelines/testing_utils/__init__.py @@ -1,4 +1,5 @@ from .common import ( + OPTIONAL_PARAMS, BaseModularPipelineOutputMixin, BaseModularPipelineTesterConfig, ModularPipelineTesterMixin, @@ -17,6 +18,7 @@ __all__ = [ + "OPTIONAL_PARAMS", "BaseModularPipelineOutputMixin", "BaseModularPipelineTesterConfig", "ModularAutoOffloadTesterMixin", diff --git a/tests/modular_pipelines/testing_utils/common.py b/tests/modular_pipelines/testing_utils/common.py index 0bc62861a826..63e5c933216f 100644 --- a/tests/modular_pipelines/testing_utils/common.py +++ b/tests/modular_pipelines/testing_utils/common.py @@ -32,6 +32,12 @@ ) +# the optional inputs every pipeline is expected to take; see `BaseModularPipelineTesterConfig.optional_params` +OPTIONAL_PARAMS = frozenset( + ["num_inference_steps", "num_images_per_prompt", "latents", "output_type", "generator", "height", "width"] +) + + class BaseModularPipelineTesterConfig: """ Base class defining the configuration interface for modular pipeline testing. @@ -44,10 +50,9 @@ class BaseModularPipelineTesterConfig: """ # Canonical parameters that are passed to `__call__` regardless of the type of pipeline. They are always - # optional and have common sense default values. - optional_params = frozenset( - ["num_inference_steps", "num_images_per_prompt", "latents", "output_type", "generator", "height", "width"] - ) + # optional and have common sense default values. A pipeline that lacks or renames one composes its own set from + # the constant, e.g. `OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"}` + optional_params = OPTIONAL_PARAMS # Canonical parameters the pipeline cannot run without. `test_pipeline_required_parameters` asserts the blocks mark # them required; a pipeline that does not take a prompt overrides this with its own, e.g. `{"audio", "image"}` required_params = frozenset(["prompt"]) diff --git a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py index f95e61e361f9..20fbd07aee42 100644 --- a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py +++ b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py @@ -18,6 +18,7 @@ from diffusers.modular_pipelines import WanBlocks, WanModularPipeline from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -31,7 +32,7 @@ class WanModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = WanBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-wan-modular-pipe" expected_latents_shape = (1, 16, 3, 2, 2) - optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} output_name = "videos" def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py b/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py index 9ac353510d9b..db16bd888606 100644 --- a/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py +++ b/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py @@ -21,6 +21,7 @@ from diffusers.modular_pipelines import Wan22VaceBlocks, Wan22VaceModularPipeline from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -37,7 +38,7 @@ class Wan22VaceModularPipelineTesterConfig(BaseModularPipelineTesterConfig): # a list of prompts is not yet supported: `WanVaceAdditionalInputsStep` raises on one batch_params = frozenset() - optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents"]) + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} output_name = "videos" def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py b/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py index 21bf2f046448..c8305b187eec 100644 --- a/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py +++ b/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py @@ -25,6 +25,7 @@ ) from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularGuiderTesterMixin, ModularLoadingTesterMixin, @@ -88,7 +89,7 @@ class WanAnimate2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): # one prompt per call: the text encoder raises on a list batch_params = frozenset() - optional_params = frozenset(["num_inference_steps", "height", "width", "output_type"]) + optional_params = OPTIONAL_PARAMS - {"latents", "num_images_per_prompt"} output_name = "videos" expected_workflow_defaults = WAN_ANIMATE_2_DEFAULTS From fcb564b39f03e5c3a32737e8bf0ea249f5698732 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 17 Nov 2026 07:49:08 +0100 Subject: [PATCH 24/33] [modular] note that QwenImage's auto text encoder is kept for backward compatibility `QwenImageAutoTextEncoderStep` makes the text encoder skippable, so the preset lists `prompt` as optional. The behaviour stays; the class, the modular guide and the test config now say that new pipelines should use the text encoder step directly. Co-Authored-By: Claude Fable 5.2 --- .ai/references/modular.md | 2 +- .../modular_pipelines/qwenimage/modular_blocks_qwenimage.py | 2 ++ .../modular_pipelines/qwen/test_modular_pipeline_qwenimage.py | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.ai/references/modular.md b/.ai/references/modular.md index e418bfb4fc33..dfb7ae288be0 100644 --- a/.ai/references/modular.md +++ b/.ai/references/modular.md @@ -225,7 +225,7 @@ Two consequences for input plumbing: 1. **Encoder / VAE-encoder blocks accept raw inputs only** (`prompt`, `image`, ...) and emit per-prompt outputs (`prompt_embeds`, `image_latents`). They do **not** bake in `num_images_per_prompt`. 2. **Per-prompt expansion happens in a dedicated input step** inside the core denoise sequence (e.g. `TextInputStep`). That keeps pre-encoded embeds reusable across runs with different `num_images_per_prompt`. See `qwenimage/before_denoise.py` for the canonical input step. -Standard pipelines accept `prompt_embeds` / `image_latents` as `__call__` inputs so users can skip encoding. In modular pipelines this is unnecessary — users just pop out the encoder block and run it standalone. Don't accept pre-computed encoder outputs as `__call__` inputs of an encoder block. +Standard pipelines accept `prompt_embeds` / `image_latents` as `__call__` inputs so users can skip encoding. In modular pipelines this is unnecessary — users just pop out the encoder block and run it standalone. Don't accept pre-computed encoder outputs as `__call__` inputs of an encoder block. For the same reason, don't wrap the text encoder in an `AutoPipelineBlocks` triggered by `prompt` so that it can be skipped. `QwenImageAutoTextEncoderStep` does this and is kept only for backward compatibility. ## Key pattern: Flat blocksets diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py index 1e3d63bb1bbd..34ce5c72b878 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py @@ -60,6 +60,8 @@ # ==================== +# Kept for backward compatibility, do not copy this for new pipelines: wrapping the text encoder in an auto block makes +# it skippable. Use the text encoder step directly, a user who has embeddings can drop the block # auto_docstring class QwenImageAutoTextEncoderStep(AutoPipelineBlocks): """ diff --git a/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py b/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py index e7fe9f5b0033..626455d9cae5 100644 --- a/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py +++ b/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py @@ -139,6 +139,9 @@ class QwenImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = QwenImageAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage-modular" expected_latents_shape = (1, 4, 1, 8, 8) + # `QwenImageAutoTextEncoderStep` skips the text encoder when no `prompt` is passed, so the preset lists `prompt` as + # optional. Kept for backward compatibility, new pipelines should not copy this + required_params = frozenset() batch_params = frozenset(["prompt", "negative_prompt", "image", "mask_image"]) expected_workflow_blocks = QWEN_IMAGE_TEXT2IMAGE_WORKFLOWS From 6dec230a9366b73651d6ce2a2db9cfecfb4c4564 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 17 Nov 2026 07:49:08 +0100 Subject: [PATCH 25/33] [modular] a skippable block that rewrites its own input does not supply it LTX2's prompt enhancer requires `prompt` and writes `prompt`, so the walk treated `prompt` as maybe produced and made the text encoder's requirement optional. The walk now tracks maybe-outputs whose producing blocks all require the same name as an input, and those no longer relax a later requirement. Co-Authored-By: Claude Fable 5.2 --- .../ltx2/modular_blocks_ltx2.py | 10 +- .../ltx2/modular_blocks_ltx25.py | 2 +- .../modular_pipelines/modular_pipeline.py | 35 ++++++- .../test_sequential_pipeline_blocks.py | 92 +++++++++++++++++++ 4 files changed, 128 insertions(+), 11 deletions(-) diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py index 6da52ba54d88..86022f8cbd3e 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx2.py @@ -1268,7 +1268,7 @@ class LTX2Blocks(SequentialPipelineBlocks): (`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, depends on the workflow): + prompt (`str`): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1380,7 +1380,7 @@ class LTX2ImageToVideoBlocks(SequentialPipelineBlocks): audio_guider (`LTX2Guidance`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, depends on the workflow): + prompt (`str`): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1501,7 +1501,7 @@ class LTX2ConditionBlocks(SequentialPipelineBlocks): (`LTX2Guidance`) video_processor (`VideoProcessor`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, depends on the workflow): + prompt (`str`): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1621,7 +1621,7 @@ class LTX2InContextBlocks(SequentialPipelineBlocks): (`LTX2Guidance`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, depends on the workflow): + prompt (`str`): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the @@ -1777,7 +1777,7 @@ class LTX2AutoBlocks(SequentialPipelineBlocks): audio_guider (`LTX2Guidance`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, depends on the workflow): + prompt (`str`): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the diff --git a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py index 1219b9a8d937..fb0dc64e4ab1 100644 --- a/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py +++ b/src/diffusers/modular_pipelines/ltx2/modular_blocks_ltx25.py @@ -233,7 +233,7 @@ class LTX25AutoBlocks(SequentialPipelineBlocks): audio_guider (`LTX2Guidance`) diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) vocoder (`LTX2Vocoder`) Inputs: - prompt (`str`, *optional*, depends on the workflow): + prompt (`str`): The prompt or prompts to guide image generation. conditions (`list`, *optional*): `LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index e8b6023f45f9..2409006595e7 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -1372,8 +1372,13 @@ def _get_inputs(self): inputs = {} # input_name -> InputParam, e.g. {"num_inference_steps": InputParam("num_inference_steps", default=50)} outputs = set() # produced by an earlier block # maybe produced by an earlier block: a ConditionalPipelineBlocks that can be skipped, or whose branches do not - # all produce the output, at any depth + # all produce the output, at any depth. e.g. `image_latents` from a skippable `image_encoder` maybe_outputs = set() + # could be produced by an earlier block, but only when the block receives the same input. + # `requires_same_input` is used to determine if an output can stand in for an input the caller did not pass, + # e.g. a skippable prompt enhancer writes `prompt` but also requires `prompt` as an input, so it cannot stand + # in for a `prompt` the caller did not pass + requires_same_input = set() for block_name, block in self.sub_blocks.items(): for inp in block.inputs: # kwargs inputs have no name, key them by "*_" @@ -1393,7 +1398,7 @@ def _get_inputs(self): inputs[key].defaults_by_block = { f"{block_name}.{k}": v for k, v in inp.defaults_by_block.items() } - if key in maybe_outputs: + if key in maybe_outputs and key not in requires_same_input: # an earlier conditional block can produce it, so the caller does not have to pass it even if # this block requires it. # e.g. `prepare_latents` lists `image_latents` as a required input, but `image_encoder` (a @@ -1415,7 +1420,7 @@ def _get_inputs(self): has_default = existing.default is not None or any( d is not None and d is not REQUIRED for d in (existing.defaults_by_block or {}).values() ) - if key not in maybe_outputs and not has_default: + if not (key in maybe_outputs and key not in requires_same_input) and not has_default: existing.required = True elif ( not inp.required @@ -1439,9 +1444,29 @@ def _get_inputs(self): f"earlier declaration." ) # an output the block always produces goes to `outputs`, one marked `_maybe` (see `OutputParam`) to - # `maybe_outputs`, e.g. `image_encoder` may be skipped, so `image_latents` is only produced if it runs + # `maybe_outputs`, e.g. `image_encoder` may be skipped, so `image_latents` is only produced if it runs. + # A maybe-output also goes to `requires_same_input` unless the block is able to produce it without + # requiring it as an input: we look through its sub-blocks, at any depth, for one that does for out in block.intermediate_outputs: - (maybe_outputs if out._maybe else outputs).add(out.name) + if not out._maybe: + outputs.add(out.name) + continue + to_visit = [block] + produced_on_its_own = False + while to_visit and not produced_on_its_own: + current = to_visit.pop() + if isinstance(current, (SequentialPipelineBlocks, ConditionalPipelineBlocks)): + to_visit.extend(current.sub_blocks.values()) + else: + produced_on_its_own = out.name in {o.name for o in current.intermediate_outputs} and ( + out.name not in {i.name for i in current.inputs if i.required} + ) + if produced_on_its_own: + # another block can produce it on its own, so it can stand in for the input + requires_same_input.discard(out.name) + elif out.name not in maybe_outputs: + requires_same_input.add(out.name) + maybe_outputs.add(out.name) return list(inputs.values()) diff --git a/tests/modular_pipelines/test_sequential_pipeline_blocks.py b/tests/modular_pipelines/test_sequential_pipeline_blocks.py index f76ec130942a..98fd7345fd44 100644 --- a/tests/modular_pipelines/test_sequential_pipeline_blocks.py +++ b/tests/modular_pipelines/test_sequential_pipeline_blocks.py @@ -313,6 +313,98 @@ def __call__(self, components, state): assert required_flags(blocks, "image", "image_latents") == {"image": False, "image_latents": False} assert blocks.required_inputs == [] + def test_skippable_block_that_rewrites_its_own_input_does_not_supply_it(self): + class EnhancePromptBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("prompt", required=True), InputParam("enhance", default=False)] + + @property + def intermediate_outputs(self): + return [OutputParam("prompt")] + + def __call__(self, components, state): + return components, state + + class SkippableEnhance(ConditionalPipelineBlocks): + block_classes = [EnhancePromptBlock] + block_names = ["enhance"] + block_trigger_inputs = ["enhance"] + default_block_name = None + + def select_block(self, enhance=None): + return "enhance" if enhance else None + + class EncodePromptBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("prompt", required=True)] + + def __call__(self, components, state): + return components, state + + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"enhance": SkippableEnhance(), "encode": EncodePromptBlock()} + ) + # the enhancer only rewrites the `prompt` it was given: skipped, nothing produces it; run, the caller passed + # it. So the encoder's requirement stands and `prompt` is a required input + assert required_flags(blocks, "prompt") == {"prompt": True} + assert blocks.required_inputs == ["prompt"] + + # same answer when the enhancer sits one level down, inside another sequence + nested = SequentialPipelineBlocks.from_blocks_dict( + { + "prepare": SequentialPipelineBlocks.from_blocks_dict({"enhance": SkippableEnhance()}), + "encode": EncodePromptBlock(), + } + ) + assert required_flags(nested, "prompt") == {"prompt": True} + + def test_skippable_block_that_reads_its_output_optionally_supplies_it(self): + class ResolveSizeBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("image", required=True), InputParam("height")] + + @property + def intermediate_outputs(self): + return [OutputParam("height")] + + def __call__(self, components, state): + return components, state + + class SkippableResolveSize(ConditionalPipelineBlocks): + block_classes = [ResolveSizeBlock] + block_names = ["resolve"] + block_trigger_inputs = ["image"] + default_block_name = None + + def select_block(self, image=None): + return "resolve" if image is not None else None + + class NeedsHeightBlock(ModularPipelineBlocks): + model_name = "test" + + @property + def inputs(self): + return [InputParam("height", required=True)] + + def __call__(self, components, state): + return components, state + + blocks = SequentialPipelineBlocks.from_blocks_dict( + {"resolve": SkippableResolveSize(), "consume": NeedsHeightBlock()} + ) + # `resolve` reads `height` only optionally and works it out from `image` otherwise, so it can supply it: the + # caller passes `image` or `height` + assert required_flags(blocks, "image", "height") == {"image": False, "height": False} + def test_output_of_some_branches_only_keeps_required_input_listed_but_optional(self): class EncodeImageAndCountBlock(EncodeImageBlock): @property From 0556f3dcc20aa4f45902f216619c28cce33b3b98 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 17 Nov 2026 08:55:02 +0100 Subject: [PATCH 26/33] [docs] modular guide: `new()` gets its own section, composite declarations trimmed Co-Authored-By: Claude Fable 5.2 --- .ai/references/modular.md | 53 ++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/.ai/references/modular.md b/.ai/references/modular.md index dfb7ae288be0..1f59ed332eb6 100644 --- a/.ai/references/modular.md +++ b/.ai/references/modular.md @@ -282,14 +282,7 @@ if block_state.num_frames is None: block_state.num_frames = 189 ``` -A declared default is part of the block's contract, so the assembled pipeline is aware of it. The generated docstring shows it: - -``` -num_frames (`int`, *optional*, defaults to 189): - Number of frames to generate. -``` - -and `default_call_parameters` reports it. If you resolve it inside the body instead, the input renders as `*optional*` with no default, and nothing at the pipeline level can report what the block will actually do. Resolve inside `__call__` only when the default is *computed*, derived from other inputs or component config: +A declared default is part of the block's contract: it shows in the generated docstring and in `default_call_parameters`; one resolved in the body does not. Resolve inside `__call__` only when the default is *computed*, derived from other inputs or component config: ```python if block_state.height is None: @@ -300,25 +293,13 @@ The other two ways to declare an input: (1) `required=True` when the block cannot work without the value, so the caller has to pass it; the block raises `ValueError("Required input 'x' is missing")` when it is not passed or `None`. (2) Optional with no default when the block works with `None`. Most often that is the computed case above, and the block should then also declare the value as an output. -**An assembled block derives its declarations from its sub-blocks; to change one, change the sub-block with `new()`.** Inputs, outputs, components and configs are declared on leaf blocks. A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its own from its sub-blocks and cannot add, remove or redefine any of them. To reuse a leaf block where only what it declares differs (the default or description of an input, the repo and loading info of a component, the value of a config) create it with `new()`: `block.new(description=..., inputs=[...], intermediate_outputs=[...], outputs=[...], components=[...], configs=[...])` returns a copy of the block with the listed entries replaced by name. It cannot add or remove an entry, and it only works on a block without sub-blocks. The block's code is unchanged, so a block that reads other inputs or runs other code is a block of its own, not a `new()` of another (gotcha #13). `intermediate_outputs` of a composite has no override: a composite can define the `outputs` property to choose the ones it lists and how it describes them. +**A composite block derives its declarations from its sub-blocks; don't change them on the composite.** A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its inputs, intermediate outputs, components and configs from its sub-blocks, so don't override `inputs`, `intermediate_outputs`, `expected_components` or `expected_configs` on it. Instead, create the sub-block with the declaration you need using `new()` (see "Reusing a block with `new()`" below). `outputs` is the exception, see below. -```python -class DistilledCoreDenoiseStep(SequentialPipelineBlocks): - block_classes = [ - # The distilled checkpoint samples in few steps. - SetTimestepsStep().new(inputs=[InputParam.template("num_inference_steps", default=10)]), - ..., - ] - block_names = [...] -``` +How the composite derives a shared input: -`components=` and `configs=` work the same way with a `ComponentSpec` / `ConfigSpec`, see the ComponentSpec section below. Typical uses: set a preset's default as above, settle a default two sub-blocks disagree on, or reword a description. - -How the composite derives a shared input, and when you need an override: - -- In a conditional block, the sub-blocks may declare different defaults for the same input, and each is respected: e.g. for `strength`, the img2img branch defaults to 0.3 and the inpaint branch to 0.9999, the docstring of the conditional block says "depends on the workflow", and at runtime the branch that runs fills in its own. To give every branch one value, use `new()` on the branch blocks that declare it. -- In a sequential block, the first sub-block that declares a default sets it for every sub-block after it. When a later sub-block declares a different default, the sequence keeps the first one and logs the other at debug level; make the two sub-blocks agree, with `new()` on one of them if the shared block is right elsewhere. A later default after a sub-block that leaves the input open (no default) is not a conflict: the sequence keeps the input open. -- The `required` flag the composite reports is its own view, whether the caller has to pass the value. In a sequential block, a sub-block may require an input that an earlier sub-block defaults or may produce; the composite then lists it as optional. In a conditional block, an input is required only when every branch requires it and the block always runs. The sub-blocks' own declarations are untouched and still checked at runtime in `get_block_state`. +- In a sequential block, the first sub-block that declares a default sets it for every sub-block after it. If two sub-blocks declare different defaults, make them agree, with `new()` on one of them. +- In a conditional block, each branch keeps its own default, and the docstring says "depends on the workflow". +- An input a sub-block requires is listed as optional on the composite when an earlier sub-block gives it a default or may produce it. **`outputs` is what the assembled block presents as its product.** By default it is every intermediate output. Override the property to list only what the step is for, so the docstring shows the product instead of every internal intermediate (Wan-Animate-2's core denoise lists only `segment_frames`). It changes the docstring and `output_names` only: everything stays in the pipeline state, and `pipe(..., output=)` still fetches any intermediate, listed or not. @@ -338,13 +319,19 @@ ComponentSpec( ) ``` -A composite inherits its sub-blocks' `ComponentSpec`s and `ConfigSpec`s. To point one at another repo, loading method or config value for the assembled pipeline, create the sub-block that declares it with `new(components=[...])` / `new(configs=[...])`, the same way `new(inputs=[...])` redeclares an input (see InputParam / OutputParam above). +A composite inherits its sub-blocks' `ComponentSpec`s and `ConfigSpec`s. To point one at another repo, loading method or config value for the assembled pipeline, see "Reusing a block with `new()`" below. + +## Reusing a block with `new()` + +A common case: a block does what you need, but one of its declarations does not fit, e.g. the default or description of an input, the repo and loading info of a component, the value of a config. Don't copy or subclass the block for that, create it with `new()`: ```python class DistilledBlocks(SequentialPipelineBlocks): block_classes = [ TextEncoderStep, - # the distilled checkpoint has its own transformer weights and skips guidance + # the distilled checkpoint samples in few steps + SetTimestepsStep().new(inputs=[InputParam.template("num_inference_steps", default=10)]), + # it has its own transformer weights and skips guidance DenoiseStep().new( components=[ ComponentSpec("transformer", YourTransformerModel, repo="org/model-distilled", subfolder="transformer") @@ -353,9 +340,17 @@ class DistilledBlocks(SequentialPipelineBlocks): ), DecodeStep, ] - block_names = ["text_encoder", "denoise", "decode"] + block_names = ["text_encoder", "set_timesteps", "denoise", "decode"] ``` +`block.new(description=..., inputs=[...], intermediate_outputs=[...], outputs=[...], components=[...], configs=[...])` returns a copy of the block with the listed entries replaced by name. Typical uses: set a preset's default or checkpoint as above, settle a default two sub-blocks disagree on, or reword a description. + +What `new()` does not do: + +- It cannot add or remove an entry, only replace one the block already declares. +- It only works on a block without sub-blocks. +- It does not change the block's code. A block that reads other inputs or runs other code is a block of its own, not a `new()` of another (gotcha #13). + ## Gotchas 1. **Importing from standard pipelines.** The modular and standard pipeline systems are parallel — modular blocks must not import from `diffusers.pipelines.*`. For shared utility methods (e.g. `_pack_latents`, `retrieve_timesteps`), either redefine as standalone functions or use `# Copied from diffusers.pipelines....` headers. See `wan/before_denoise.py` and `helios/before_denoise.py` for examples. @@ -385,7 +380,7 @@ class DistilledBlocks(SequentialPipelineBlocks): 12. **Helpers taking `components`.** Prefer explicit args, for example, `get_xx_prompt_embeds(text_encoder, tokenizer, prompt)` instead of `get_xx_prompt_embeds(components, prompt)`. Helpers and block methods should take only what they use so dependencies are visible and they can run without an assembled pipeline. Leave the full `components` object to `__call__`, which unpacks what its helpers need. See `get_qwen_prompt_embeds` in `qwenimage/encoders.py`. -13. **A `new()` entry that restates less than the block's own.** `block.new(inputs=[InputParam("num_inference_steps", default=10)])` replaces the entry whole, so the type hint and description the block declared are gone. Build it from the template, `InputParam.template("num_inference_steps", default=10)`, or restate everything. And `new()` is only for declarations: a block that reads other inputs or runs other code is written as its own class, not as a subclass or a `new()` of another. +13. **A `new()` entry that restates less than the block's own.** `block.new(inputs=[InputParam("num_inference_steps", default=10)])` replaces the entry whole, so the type hint and description the block declared are gone. Build it from the template, `InputParam.template("num_inference_steps", default=10)`, or restate everything. ## Conversion checklist From ddbc216c26690e6cb3096711f5f16c97761023b3 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 17 Nov 2026 09:05:10 +0100 Subject: [PATCH 27/33] [docs] modular: simplify the composite declarations and conflicting defaults paragraphs Co-Authored-By: Claude Fable 5.2 --- .ai/references/modular.md | 10 +++------- .../en/modular_diffusers/sequential_pipeline_blocks.md | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.ai/references/modular.md b/.ai/references/modular.md index 1f59ed332eb6..2dc4ffcb8c3d 100644 --- a/.ai/references/modular.md +++ b/.ai/references/modular.md @@ -293,15 +293,11 @@ The other two ways to declare an input: (1) `required=True` when the block cannot work without the value, so the caller has to pass it; the block raises `ValueError("Required input 'x' is missing")` when it is not passed or `None`. (2) Optional with no default when the block works with `None`. Most often that is the computed case above, and the block should then also declare the value as an output. -**A composite block derives its declarations from its sub-blocks; don't change them on the composite.** A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its inputs, intermediate outputs, components and configs from its sub-blocks, so don't override `inputs`, `intermediate_outputs`, `expected_components` or `expected_configs` on it. Instead, create the sub-block with the declaration you need using `new()` (see "Reusing a block with `new()`" below). `outputs` is the exception, see below. +A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its inputs, intermediate outputs, components and configs from its sub-blocks, so don't override `inputs`, `intermediate_outputs`, `expected_components` or `expected_configs` on it. Instead, create the sub-block with the declaration you need using `new()` (see "Reusing a block with `new()`" below). -How the composite derives a shared input: +The exception is `outputs`, which is what the composite presents as its product. By default it is every intermediate output. Override the property to list only what the step is for, so the docstring shows the product instead of every internal intermediate (Wan-Animate-2's core denoise lists only `segment_frames`). It changes the docstring and `output_names` only: everything stays in the pipeline state, and `pipe(..., output=)` still fetches any intermediate, listed or not. -- In a sequential block, the first sub-block that declares a default sets it for every sub-block after it. If two sub-blocks declare different defaults, make them agree, with `new()` on one of them. -- In a conditional block, each branch keeps its own default, and the docstring says "depends on the workflow". -- An input a sub-block requires is listed as optional on the composite when an earlier sub-block gives it a default or may produce it. - -**`outputs` is what the assembled block presents as its product.** By default it is every intermediate output. Override the property to list only what the step is for, so the docstring shows the product instead of every internal intermediate (Wan-Animate-2's core denoise lists only `segment_frames`). It changes the docstring and `output_names` only: everything stays in the pipeline state, and `pipe(..., output=)` still fetches any intermediate, listed or not. +When two sub-blocks of a sequence declare different defaults for the same input, the sequence uses the first one; make sure every sub-block has the intended default, with `new()` where it does not. An input a sub-block requires can show as optional on the composite, when an earlier sub-block gives it a default or may produce it; that is expected. ## ComponentSpec patterns diff --git a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md index c472aa06cf52..30e18d696a9a 100644 --- a/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md +++ b/docs/source/en/modular_diffusers/sequential_pipeline_blocks.md @@ -132,7 +132,7 @@ class ImageProcessingStep(SequentialPipelineBlocks): block_names = ["input", "image_encoder"] ``` -When two sub-blocks declare different defaults for the same input, the sequence keeps the first one (the other is logged at debug level). Make them agree: `new()` on the block whose default is right elsewhere gives it the value this sequence needs. A later default after a sub-block that leaves the input open is not a conflict; the sequence keeps the input open. +When two sub-blocks declare different defaults for the same input, the sequence uses the first one. To remove the ambiguity, make sure every sub-block that declares the input has the intended default, creating it with `new()` where it does not. A few properties you set yourself: From 588c282dd57163bcc92647aad39145c8da935e56 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Wed, 18 Nov 2026 03:03:24 +0100 Subject: [PATCH 28/33] [modular tests] pipeline-specific inputs go into `optional_params` or `required_params` The names each test config used to list in `params` are asserted again: the ones the blocks mark required in `required_params`, the rest in `optional_params`, on top of the shared defaults. Co-Authored-By: Claude Fable 5.2 --- .../anima/test_modular_pipeline_anima.py | 3 +++ .../cosmos/test_modular_pipeline_cosmos3.py | 2 +- .../cosmos/test_modular_pipeline_cosmos3_distilled.py | 2 +- .../ernie_image/test_modular_pipeline_ernie_image.py | 2 ++ .../modular_pipelines/flux/test_modular_pipeline_flux.py | 4 ++++ .../flux2/test_modular_pipeline_flux2.py | 3 +++ .../flux2/test_modular_pipeline_flux2_klein.py | 3 +++ .../flux2/test_modular_pipeline_flux2_klein_base.py | 3 +++ .../helios/test_modular_pipeline_helios.py | 3 ++- .../test_modular_pipeline_hunyuan_video1_5.py | 2 +- .../kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py | 2 +- tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py | 2 +- .../modular_pipelines/ltx2/test_modular_pipeline_ltx2.py | 8 +++++++- .../modular_pipelines/ltx2/test_modular_pipeline_ltx25.py | 8 +++++++- .../minimax_h3/test_modular_pipeline_minimax_h3.py | 4 ++-- .../test_modular_pipeline_minimax_h3_image.py | 2 +- .../test_modular_pipeline_minimax_music3.py | 3 ++- .../p_h3_image/test_modular_pipeline_p_h3_image.py | 2 +- .../modular_pipelines/pan2/test_modular_pipeline_pan2.py | 2 +- .../pika_h3_flex/test_modular_pipeline_pika_h3_flex.py | 8 ++++++-- .../qwen/test_modular_pipeline_qwenimage.py | 6 ++++++ .../qwenimage21/test_modular_pipeline_qwenimage21.py | 2 ++ .../test_modular_pipeline_qwenimage21_turbo.py | 2 ++ .../test_modular_pipeline_stable_diffusion_3.py | 2 ++ .../test_modular_pipeline_stable_diffusion_xl.py | 4 ++++ tests/modular_pipelines/wan/test_modular_pipeline_wan.py | 2 +- .../wan/test_modular_pipeline_wan_vace.py | 8 +++++++- .../wan_animate_2/test_modular_pipeline_wan_animate_2.py | 1 + 28 files changed, 77 insertions(+), 18 deletions(-) diff --git a/tests/modular_pipelines/anima/test_modular_pipeline_anima.py b/tests/modular_pipelines/anima/test_modular_pipeline_anima.py index 681c3b2b13e4..99548894993d 100644 --- a/tests/modular_pipelines/anima/test_modular_pipeline_anima.py +++ b/tests/modular_pipelines/anima/test_modular_pipeline_anima.py @@ -30,6 +30,7 @@ from ...testing_utils import enable_full_determinism, is_lora, require_peft_backend from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineOutputMixin, BaseModularPipelineTesterConfig, ModularGuiderTesterMixin, @@ -111,6 +112,7 @@ class AnimaModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = AnimaAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-anima-modular-pipe" expected_latents_shape = (1, 4, 1, 8, 8) + optional_params = OPTIONAL_PARAMS | {"negative_prompt"} batch_params = frozenset(["prompt", "negative_prompt"]) expected_workflow_blocks = ANIMA_TEXT2IMAGE_WORKFLOWS @@ -256,6 +258,7 @@ class AnimaImg2ImgModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = AnimaAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-anima-modular-pipe" expected_latents_shape = (1, 4, 1, 8, 8) + optional_params = OPTIONAL_PARAMS | {"image", "negative_prompt", "strength"} batch_params = frozenset(["prompt", "negative_prompt"]) expected_workflow_blocks = ANIMA_IMG2IMG_WORKFLOWS diff --git a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py index eeed6f133251..0feb240c3b18 100644 --- a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py +++ b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3.py @@ -136,7 +136,7 @@ class Cosmos3OmniModularPipelineTesterConfig(BaseModularPipelineTesterConfig): expected_latents_shape = (1, 4, 2, 4, 4) # one prompt per call: the text encoder raises on a list batch_params = frozenset() - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"guidance_scale", "num_frames"} output_name = "videos" expected_workflow_blocks = COSMOS3_OMNI_WORKFLOWS diff --git a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py index c38c4cd40fce..55280af95a58 100644 --- a/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py +++ b/tests/modular_pipelines/cosmos/test_modular_pipeline_cosmos3_distilled.py @@ -75,7 +75,7 @@ class Cosmos3DistilledModularPipelineTesterConfig(BaseModularPipelineTesterConfi expected_latents_shape = (1, 4, 2, 4, 4) # one prompt per call: the text encoder raises on a list batch_params = frozenset() - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_frames"} output_name = "videos" expected_workflow_blocks = COSMOS3_DISTILLED_WORKFLOWS diff --git a/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py b/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py index 24170eaf310b..de980912b812 100644 --- a/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py +++ b/tests/modular_pipelines/ernie_image/test_modular_pipeline_ernie_image.py @@ -18,6 +18,7 @@ from diffusers.modular_pipelines import ErnieImageAutoBlocks, ErnieImageModularPipeline from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -43,6 +44,7 @@ class ErnieImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = ErnieImageAutoBlocks pretrained_model_name_or_path = "akshan-main/tiny-ernie-image-modular-pipe" expected_latents_shape = (1, 16, 2, 2) + optional_params = OPTIONAL_PARAMS | {"prompt"} # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() expected_workflow_blocks = ERNIE_IMAGE_WORKFLOWS diff --git a/tests/modular_pipelines/flux/test_modular_pipeline_flux.py b/tests/modular_pipelines/flux/test_modular_pipeline_flux.py index 37c494ea546e..5dfef7792f4c 100644 --- a/tests/modular_pipelines/flux/test_modular_pipeline_flux.py +++ b/tests/modular_pipelines/flux/test_modular_pipeline_flux.py @@ -31,6 +31,7 @@ from ...testing_utils import floats_tensor, torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularAutoOffloadTesterMixin, ModularLoadingTesterMixin, @@ -59,6 +60,7 @@ class FluxModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = FluxAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux-modular" expected_latents_shape = (1, 1, 8, 8) + optional_params = OPTIONAL_PARAMS | {"guidance_scale"} expected_workflow_blocks = FLUX_TEXT2IMAGE_WORKFLOWS @@ -134,6 +136,7 @@ class FluxImg2ImgModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = FluxAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux-modular" expected_latents_shape = (1, 1, 8, 8) + optional_params = OPTIONAL_PARAMS | {"guidance_scale", "image"} batch_params = frozenset(["prompt", "image"]) expected_workflow_blocks = FLUX_IMAGE2IMAGE_WORKFLOWS @@ -233,6 +236,7 @@ class FluxKontextModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = FluxKontextAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux-kontext-pipe" expected_latents_shape = (1, 1, 8, 8) + optional_params = OPTIONAL_PARAMS | {"guidance_scale", "image"} batch_params = frozenset(["prompt", "image"]) expected_workflow_blocks = FLUX_KONTEXT_WORKFLOWS diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py index a2a05a1ecd3a..ab869c0c9fa7 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2.py @@ -26,6 +26,7 @@ from ...testing_utils import floats_tensor, torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -54,6 +55,7 @@ class Flux2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = Flux2AutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-modular" expected_latents_shape = (1, 4, 16, 16) + optional_params = OPTIONAL_PARAMS | {"guidance_scale", "prompt"} # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() expected_workflow_blocks = FLUX2_TEXT2IMAGE_WORKFLOWS @@ -115,6 +117,7 @@ class Flux2ImageConditionedModularPipelineTesterConfig(BaseModularPipelineTester pipeline_blocks_class = Flux2AutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-modular" expected_latents_shape = (1, 4, 16, 16) + optional_params = OPTIONAL_PARAMS | {"guidance_scale", "image", "prompt"} # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() batch_params = frozenset(["prompt", "image"]) diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py index aba1992384bc..9249c4d3208c 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein.py @@ -26,6 +26,7 @@ from ...testing_utils import floats_tensor, torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -53,6 +54,7 @@ class Flux2KleinModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = Flux2KleinAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-modular" expected_latents_shape = (1, 4, 16, 16) + optional_params = OPTIONAL_PARAMS | {"prompt"} # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() not_params = frozenset(["negative_prompt"]) @@ -113,6 +115,7 @@ class Flux2KleinImageConditionedModularPipelineTesterConfig(BaseModularPipelineT pipeline_blocks_class = Flux2KleinAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-modular" expected_latents_shape = (1, 4, 16, 16) + optional_params = OPTIONAL_PARAMS | {"image", "prompt"} # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() batch_params = frozenset(["prompt", "image"]) diff --git a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py index 20c92b6801ab..3f94cde6a3d2 100644 --- a/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py +++ b/tests/modular_pipelines/flux2/test_modular_pipeline_flux2_klein_base.py @@ -26,6 +26,7 @@ from ...testing_utils import floats_tensor, torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -53,6 +54,7 @@ class Flux2KleinBaseModularPipelineTesterConfig(BaseModularPipelineTesterConfig) pipeline_blocks_class = Flux2KleinBaseAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-base-modular" expected_latents_shape = (1, 4, 16, 16) + optional_params = OPTIONAL_PARAMS | {"prompt"} # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() expected_workflow_blocks = FLUX2_KLEIN_BASE_WORKFLOWS @@ -112,6 +114,7 @@ class Flux2KleinBaseImageConditionedModularPipelineTesterConfig(BaseModularPipel pipeline_blocks_class = Flux2KleinBaseAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-flux2-klein-base-modular" expected_latents_shape = (1, 4, 16, 16) + optional_params = OPTIONAL_PARAMS | {"image", "prompt"} # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() batch_params = frozenset(["prompt", "image"]) diff --git a/tests/modular_pipelines/helios/test_modular_pipeline_helios.py b/tests/modular_pipelines/helios/test_modular_pipeline_helios.py index 5519caea3d5d..144651a27768 100644 --- a/tests/modular_pipelines/helios/test_modular_pipeline_helios.py +++ b/tests/modular_pipelines/helios/test_modular_pipeline_helios.py @@ -73,7 +73,7 @@ class HeliosModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = HeliosAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-helios-modular-pipe" expected_latents_shape = (1, 16, 9, 2, 2) - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_frames", "num_videos_per_prompt"} output_name = "videos" expected_workflow_blocks = HELIOS_WORKFLOWS @@ -149,6 +149,7 @@ class HeliosPyramidModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = "hf-internal-testing/tiny-helios-pyramid-modular-pipe" expected_latents_shape = (1, 16, 9, 8, 8) optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} | { + "num_frames", "num_videos_per_prompt", "pyramid_num_inference_steps_list", } diff --git a/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py b/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py index e34e952cec01..48201d8c2fd7 100644 --- a/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py +++ b/tests/modular_pipelines/hunyuan_video1_5/test_modular_pipeline_hunyuan_video1_5.py @@ -57,7 +57,7 @@ class HunyuanVideo15ModularPipelineTesterConfig(BaseModularPipelineTesterConfig) expected_latents_shape = (1, 32, 3, 2, 2) # the text encoder encodes an empty prompt when none is given, so no input is required required_params = frozenset() - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_frames", "num_videos_per_prompt", "prompt"} expected_workflow_blocks = HUNYUANVIDEO15_WORKFLOWS output_name = "videos" diff --git a/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py b/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py index 075ca1c56632..cf87bad1643d 100644 --- a/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py +++ b/tests/modular_pipelines/kandinsky_s2v/test_modular_pipeline_kandinsky_s2v.py @@ -73,7 +73,7 @@ class KandinskyS2VModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pretrained_model_name_or_path = TINY_REPO # one prompt per call: batching is not supported batch_params = frozenset() - optional_params = OPTIONAL_PARAMS - {"latents", "num_images_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"latents", "num_images_per_prompt"} | {"num_latent_frames", "prompt"} required_params = frozenset(["audio", "image"]) output_name = "videos" latents_output_name = "latent_chunks" diff --git a/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py b/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py index 3307aba2a29d..2a438884f898 100644 --- a/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py +++ b/tests/modular_pipelines/ltx/test_modular_pipeline_ltx.py @@ -59,7 +59,7 @@ class LTXModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = LTXAutoBlocks pretrained_model_name_or_path = "akshan-main/tiny-ltx-modular-pipe" expected_latents_shape = (1, 8, 5, 16, 16) - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_frames", "num_videos_per_prompt"} expected_workflow_blocks = LTX_WORKFLOWS output_name = "videos" diff --git a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py index d273e3f99510..9062f4fb0bca 100644 --- a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py +++ b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx2.py @@ -105,7 +105,7 @@ class LTX2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_class = LTX2ModularPipeline pipeline_blocks_class = LTX2AutoBlocks pretrained_model_name_or_path = LTX2_REPO_ID - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_frames", "num_videos_per_prompt"} expected_workflow_blocks = LTX2_WORKFLOWS output_name = "videos" @@ -213,6 +213,8 @@ class TestLTX2ModularPipelineWorkflow(LTX2ModularPipelineTesterConfig, ModularWo class LTX2Image2VideoModularPipelineTesterConfig(LTX2ModularPipelineTesterConfig): + optional_params = LTX2ModularPipelineTesterConfig.optional_params | {"image"} + def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) rng = np.random.default_rng(seed) @@ -238,6 +240,8 @@ class TestLTX2Image2VideoModularPipelineMemory(LTX2Image2VideoModularPipelineTes class LTX2ConditionModularPipelineTesterConfig(LTX2ModularPipelineTesterConfig): + optional_params = LTX2ModularPipelineTesterConfig.optional_params | {"conditions"} + def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) image = torch.rand((1, 3, 32, 32), generator=torch.Generator("cpu").manual_seed(seed)) @@ -261,6 +265,8 @@ class TestLTX2ConditionModularPipelineMemory(LTX2ConditionModularPipelineTesterC class LTX2InContextModularPipelineTesterConfig(LTX2ModularPipelineTesterConfig): + optional_params = LTX2ModularPipelineTesterConfig.optional_params | {"reference_conditions"} + def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) video = torch.rand((1, 5, 3, 32, 32), generator=torch.Generator("cpu").manual_seed(seed)) diff --git a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py index 943664599f22..bad546f507e2 100644 --- a/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py +++ b/tests/modular_pipelines/ltx2/test_modular_pipeline_ltx25.py @@ -103,7 +103,7 @@ class LTX25ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_class = LTX25ModularPipeline pipeline_blocks_class = LTX25AutoBlocks pretrained_model_name_or_path = LTX25_REPO_ID - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_frames", "num_videos_per_prompt"} expected_workflow_blocks = LTX25_WORKFLOWS output_name = "videos" @@ -220,6 +220,8 @@ class TestLTX25ModularPipelineWorkflow(LTX25ModularPipelineTesterConfig, Modular class LTX25Image2VideoModularPipelineTesterConfig(LTX25ModularPipelineTesterConfig): + optional_params = LTX25ModularPipelineTesterConfig.optional_params | {"image"} + def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) rng = np.random.default_rng(seed) @@ -247,6 +249,8 @@ class TestLTX25Image2VideoModularPipelineMemory(LTX25Image2VideoModularPipelineT class LTX25ConditionModularPipelineTesterConfig(LTX25ModularPipelineTesterConfig): + optional_params = LTX25ModularPipelineTesterConfig.optional_params | {"conditions"} + def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) image = torch.rand((1, 3, 32, 32), generator=torch.Generator("cpu").manual_seed(seed)) @@ -270,6 +274,8 @@ class TestLTX25ConditionModularPipelineMemory(LTX25ConditionModularPipelineTeste class LTX25InContextModularPipelineTesterConfig(LTX25ModularPipelineTesterConfig): + optional_params = LTX25ModularPipelineTesterConfig.optional_params | {"reference_conditions"} + def get_dummy_inputs(self, seed=0): inputs = super().get_dummy_inputs(seed) video = torch.rand((1, 5, 3, 32, 32), generator=torch.Generator("cpu").manual_seed(seed)) diff --git a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py index 53a0318ba233..7ff61b8d5af5 100644 --- a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py +++ b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py @@ -177,7 +177,7 @@ class MiniMaxH3ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): batch_params = frozenset() # The checkpoint is guidance-distilled and generates a video, so neither `guidance_scale` nor # `num_images_per_prompt` exists. - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"image", "last_image", "num_frames"} output_name = "videos" expected_workflow_blocks = MINIMAX_H3_WORKFLOWS expected_workflow_defaults = MINIMAX_H3_WORKFLOW_DEFAULTS @@ -609,7 +609,7 @@ class MiniMaxH3Ref2VAModularPipelineTesterConfig(BaseModularPipelineTesterConfig batch_params = frozenset() # The checkpoint is guidance-distilled and generates a video, so neither `guidance_scale` nor # `num_images_per_prompt` exists. - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_frames", "references"} output_name = "videos" expected_workflow_blocks = MINIMAX_H3_REF2VA_WORKFLOWS expected_workflow_defaults = MINIMAX_H3_REF2VA_WORKFLOW_DEFAULTS diff --git a/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py b/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py index 0a36ca18a602..04a4de2dcd32 100644 --- a/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py +++ b/tests/modular_pipelines/minimax_h3_image/test_modular_pipeline_minimax_h3_image.py @@ -115,7 +115,7 @@ class MiniMaxH3ImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig) # One request is one packed sequence, so prompts are not batched. # one request per call: the text encoder raises on a list of prompts batch_params = frozenset() - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} | {"image"} output_name = "images" expected_workflow_blocks = MINIMAX_H3_IMAGE_WORKFLOWS expected_workflow_defaults = MINIMAX_H3_IMAGE_WORKFLOW_DEFAULTS diff --git a/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py b/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py index a11e266985a1..d053b814b93b 100644 --- a/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py +++ b/tests/modular_pipelines/minimax_music3/test_modular_pipeline_minimax_music3.py @@ -69,10 +69,11 @@ class MiniMaxMusic3ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_class = MiniMaxMusic3ModularPipeline pipeline_blocks_class = MiniMaxMusic3Blocks pretrained_model_name_or_path = "hf-internal-testing/tiny-minimax-music3-modular-pipe" + required_params = frozenset(["lyrics", "prompt"]) # The pipeline generates a single waveform per call; `prompt` and `lyrics` are single strings. # one music description per call: the encoder raises on a list batch_params = frozenset() - optional_params = OPTIONAL_PARAMS - {"height", "latents", "num_images_per_prompt", "width"} + optional_params = OPTIONAL_PARAMS - {"height", "latents", "num_images_per_prompt", "width"} | {"audio_duration"} output_name = "audios" latents_output_name = "latent_chunks" expected_latents_shape = (1, 8, 17) diff --git a/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py b/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py index 11b2070b3a80..0f430d772aa5 100644 --- a/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py +++ b/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py @@ -162,7 +162,7 @@ class PH3ImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): batch_params = frozenset() # No guidance, one image per request, fixed sampling grid: no `guidance_scale`, `num_images_per_prompt` or # `num_inference_steps`. - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} | {"image"} output_name = "images" expected_workflow_blocks = P_H3_IMAGE_WORKFLOWS expected_workflow_defaults = P_H3_IMAGE_WORKFLOW_DEFAULTS diff --git a/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py b/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py index 44afdfdf0b05..4824f15161ff 100644 --- a/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py +++ b/tests/modular_pipelines/pan2/test_modular_pipeline_pan2.py @@ -110,7 +110,7 @@ class PAN2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): expected_latents_shape = (1, 48, 2, 2, 2) # one prompt per call: the text encoder raises on a list batch_params = frozenset() - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_frames", "num_videos_per_prompt"} expected_workflow_blocks = PAN2_WORKFLOWS expected_workflow_defaults = PAN2_WORKFLOW_DEFAULTS output_name = "videos" diff --git a/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py b/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py index 9bbc64257648..0fd070f0ff77 100644 --- a/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py +++ b/tests/modular_pipelines/pika_h3_flex/test_modular_pipeline_pika_h3_flex.py @@ -161,7 +161,11 @@ class PikaH3FlexModularPipelineTesterConfig(BaseModularPipelineTesterConfig): batch_params = frozenset() # The adapters are guidance-distilled and run a fixed six-step schedule, so neither `guidance_scale` nor # `num_inference_steps` exists, and one request generates one video. - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} | { + "image", + "last_image", + "num_frames", + } not_params = frozenset(["num_inference_steps", "guidance_scale", "negative_prompt"]) output_name = "videos" expected_workflow_blocks = PIKA_H3_FLEX_WORKFLOWS @@ -391,7 +395,7 @@ class PikaH3FlexRef2VAModularPipelineTesterConfig(BaseModularPipelineTesterConfi batch_params = frozenset() # The adapters are guidance-distilled and run a fixed six-step schedule, so neither `guidance_scale` nor # `num_inference_steps` exists, and one request generates one video. - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt", "num_inference_steps"} | {"num_frames", "references"} not_params = frozenset(["num_inference_steps", "guidance_scale", "negative_prompt"]) output_name = "videos" expected_workflow_blocks = PIKA_H3_FLEX_REF2VA_WORKFLOWS diff --git a/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py b/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py index 626455d9cae5..48e3fc59a97a 100644 --- a/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py +++ b/tests/modular_pipelines/qwen/test_modular_pipeline_qwenimage.py @@ -28,6 +28,7 @@ from ...testing_utils import torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularGuiderTesterMixin, ModularLoadingTesterMixin, @@ -139,6 +140,7 @@ class QwenImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = QwenImageAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage-modular" expected_latents_shape = (1, 4, 1, 8, 8) + optional_params = OPTIONAL_PARAMS | {"attention_kwargs", "image", "mask_image", "negative_prompt", "prompt"} # `QwenImageAutoTextEncoderStep` skips the text encoder when no `prompt` is passed, so the preset lists `prompt` as # optional. Kept for backward compatibility, new pipelines should not copy this required_params = frozenset() @@ -224,6 +226,8 @@ class QwenImageEditModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = QwenImageEditAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage-edit-modular" expected_latents_shape = (1, 4, 1, 8, 8) + optional_params = OPTIONAL_PARAMS | {"attention_kwargs", "mask_image", "negative_prompt"} + required_params = frozenset(["image", "prompt"]) batch_params = frozenset(["prompt", "negative_prompt", "image", "mask_image"]) expected_workflow_blocks = QWEN_IMAGE_EDIT_WORKFLOWS @@ -272,6 +276,8 @@ class QwenImageEditPlusModularPipelineTesterConfig(BaseModularPipelineTesterConf pipeline_blocks_class = QwenImageEditPlusAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage-edit-plus-modular" expected_latents_shape = (1, 4, 1, 8, 8) + optional_params = OPTIONAL_PARAMS | {"attention_kwargs", "negative_prompt"} + required_params = frozenset(["image", "prompt"]) # No `mask_image` yet. batch_params = frozenset(["prompt", "negative_prompt", "image"]) diff --git a/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21.py b/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21.py index 4f98cacf06d5..cc6c35735b89 100644 --- a/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21.py +++ b/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21.py @@ -20,6 +20,7 @@ from ...testing_utils import assert_tensors_close, torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularGuiderTesterMixin, ModularLoadingTesterMixin, @@ -135,6 +136,7 @@ class QwenImage21ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = QwenImage21AutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage21-modular" expected_latents_shape = (1, 8, 1, 2, 2) + optional_params = OPTIONAL_PARAMS | {"attention_kwargs", "image", "negative_prompt", "output_resolution"} batch_params = frozenset(["prompt", "negative_prompt"]) expected_workflow_blocks = QWENIMAGE21_TEXT2IMAGE_WORKFLOWS expected_workflow_defaults = {"text2image": QWENIMAGE21_WORKFLOW_DEFAULTS["text2image"]} diff --git a/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py b/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py index 6f2c3638a828..cd04c6af0fe9 100644 --- a/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py +++ b/tests/modular_pipelines/qwenimage21/test_modular_pipeline_qwenimage21_turbo.py @@ -20,6 +20,7 @@ from ...testing_utils import assert_tensors_close, torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -128,6 +129,7 @@ class QwenImage21TurboModularPipelineTesterConfig(BaseModularPipelineTesterConfi pipeline_blocks_class = QwenImage21TurboAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-qwenimage21-turbo-modular" expected_latents_shape = (1, 8, 1, 2, 2) + optional_params = OPTIONAL_PARAMS | {"attention_kwargs", "image", "output_resolution"} not_params = frozenset(["negative_prompt"]) expected_workflow_blocks = QWENIMAGE21_TURBO_TEXT2IMAGE_WORKFLOWS expected_workflow_defaults = {"text2image": QWENIMAGE21_TURBO_WORKFLOW_DEFAULTS["text2image"]} diff --git a/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py b/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py index 26bfbde5539e..1ef8ffb4ee13 100644 --- a/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py +++ b/tests/modular_pipelines/stable_diffusion_3/test_modular_pipeline_stable_diffusion_3.py @@ -27,6 +27,7 @@ from ...testing_utils import floats_tensor, torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, ModularMemoryTesterMixin, @@ -138,6 +139,7 @@ class StableDiffusion3Img2ImgModularPipelineTesterConfig(BaseModularPipelineTest pipeline_blocks_class = StableDiffusion3AutoBlocks pretrained_model_name_or_path = "AlanPonnachan/tiny-sd3-modular" expected_latents_shape = (1, 4, 32, 32) + optional_params = OPTIONAL_PARAMS | {"image"} batch_params = frozenset(["prompt", "image"]) expected_workflow_blocks = SD3_IMAGE2IMAGE_WORKFLOWS diff --git a/tests/modular_pipelines/stable_diffusion_xl/test_modular_pipeline_stable_diffusion_xl.py b/tests/modular_pipelines/stable_diffusion_xl/test_modular_pipeline_stable_diffusion_xl.py index 64725f9124f9..24d27c7e9822 100644 --- a/tests/modular_pipelines/stable_diffusion_xl/test_modular_pipeline_stable_diffusion_xl.py +++ b/tests/modular_pipelines/stable_diffusion_xl/test_modular_pipeline_stable_diffusion_xl.py @@ -26,6 +26,7 @@ from ...models.unets.test_models_unet_2d_condition import create_ip_adapter_state_dict from ...testing_utils import enable_full_determinism, floats_tensor, is_ip_adapter, torch_device from ..testing_utils import ( + OPTIONAL_PARAMS, BaseModularPipelineOutputMixin, BaseModularPipelineTesterConfig, ModularGuiderTesterMixin, @@ -337,6 +338,7 @@ class SDXLModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = StableDiffusionXLAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-sdxl-modular" expected_latents_shape = (1, 4, 32, 32) + optional_params = OPTIONAL_PARAMS | {"cross_attention_kwargs", "negative_prompt"} batch_params = frozenset(["prompt", "negative_prompt"]) expected_image_output_shape = (1, 3, 64, 64) @@ -457,6 +459,7 @@ class SDXLImg2ImgModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = StableDiffusionXLAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-sdxl-modular" expected_latents_shape = (1, 4, 32, 32) + optional_params = OPTIONAL_PARAMS | {"cross_attention_kwargs", "image", "negative_prompt"} batch_params = frozenset(["prompt", "negative_prompt", "image"]) expected_image_output_shape = (1, 3, 64, 64) expected_workflow_blocks = IMAGE2IMAGE_WORKFLOWS @@ -592,6 +595,7 @@ class SDXLInpaintingModularPipelineFastTests( pipeline_blocks_class = StableDiffusionXLAutoBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-sdxl-modular" expected_latents_shape = (1, 4, 32, 32) + optional_params = OPTIONAL_PARAMS | {"cross_attention_kwargs", "image", "mask_image", "negative_prompt"} batch_params = frozenset(["prompt", "negative_prompt", "image", "mask_image"]) expected_image_output_shape = (1, 3, 64, 64) expected_workflow_blocks = INPAINTING_WORKFLOWS diff --git a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py index 20fbd07aee42..fce86fd6cb22 100644 --- a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py +++ b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py @@ -32,7 +32,7 @@ class WanModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_blocks_class = WanBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-wan-modular-pipe" expected_latents_shape = (1, 16, 3, 2, 2) - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_frames", "num_videos_per_prompt"} output_name = "videos" def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py b/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py index db16bd888606..7df7a72128d5 100644 --- a/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py +++ b/tests/modular_pipelines/wan/test_modular_pipeline_wan_vace.py @@ -38,7 +38,13 @@ class Wan22VaceModularPipelineTesterConfig(BaseModularPipelineTesterConfig): # a list of prompts is not yet supported: `WanVaceAdditionalInputsStep` raises on one batch_params = frozenset() - optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | {"num_videos_per_prompt"} + optional_params = OPTIONAL_PARAMS - {"num_images_per_prompt"} | { + "mask", + "num_frames", + "num_videos_per_prompt", + "reference_images", + "video", + } output_name = "videos" def get_dummy_inputs(self, seed=0): diff --git a/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py b/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py index c8305b187eec..589d42c87df4 100644 --- a/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py +++ b/tests/modular_pipelines/wan_animate_2/test_modular_pipeline_wan_animate_2.py @@ -85,6 +85,7 @@ class WanAnimate2ModularPipelineTesterConfig(BaseModularPipelineTesterConfig): pipeline_class = WanAnimate2ModularPipeline pipeline_blocks_class = WanAnimate2Blocks pretrained_model_name_or_path = "hf-internal-testing/tiny-wan-animate-2-modular" + required_params = frozenset(["driving_video", "image", "prompt"]) # Each segment is decoded inside the denoise loop, so no latents are left in the state. # one prompt per call: the text encoder raises on a list From 2a04bc6035c47992b7ce977237d393937788fdf4 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Fri, 20 Nov 2026 08:54:30 +0100 Subject: [PATCH 29/33] [docs] modular: where to change a default on a composite, `new()` replaces a whole entry Co-Authored-By: Claude Fable 5.2 --- .ai/references/modular.md | 14 ++++++++------ .../qwenimage/modular_blocks_qwenimage.py | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.ai/references/modular.md b/.ai/references/modular.md index 2dc4ffcb8c3d..502dddcef26e 100644 --- a/.ai/references/modular.md +++ b/.ai/references/modular.md @@ -225,7 +225,7 @@ Two consequences for input plumbing: 1. **Encoder / VAE-encoder blocks accept raw inputs only** (`prompt`, `image`, ...) and emit per-prompt outputs (`prompt_embeds`, `image_latents`). They do **not** bake in `num_images_per_prompt`. 2. **Per-prompt expansion happens in a dedicated input step** inside the core denoise sequence (e.g. `TextInputStep`). That keeps pre-encoded embeds reusable across runs with different `num_images_per_prompt`. See `qwenimage/before_denoise.py` for the canonical input step. -Standard pipelines accept `prompt_embeds` / `image_latents` as `__call__` inputs so users can skip encoding. In modular pipelines this is unnecessary — users just pop out the encoder block and run it standalone. Don't accept pre-computed encoder outputs as `__call__` inputs of an encoder block. For the same reason, don't wrap the text encoder in an `AutoPipelineBlocks` triggered by `prompt` so that it can be skipped. `QwenImageAutoTextEncoderStep` does this and is kept only for backward compatibility. +Standard pipelines accept `prompt_embeds` / `image_latents` as `__call__` inputs so users can skip encoding. In modular pipelines this is unnecessary — users just pop out the encoder block and run it standalone. Don't accept pre-computed encoder outputs as `__call__` inputs of an encoder block. ## Key pattern: Flat blocksets @@ -293,11 +293,11 @@ The other two ways to declare an input: (1) `required=True` when the block cannot work without the value, so the caller has to pass it; the block raises `ValueError("Required input 'x' is missing")` when it is not passed or `None`. (2) Optional with no default when the block works with `None`. Most often that is the computed case above, and the block should then also declare the value as an output. -A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its inputs, intermediate outputs, components and configs from its sub-blocks, so don't override `inputs`, `intermediate_outputs`, `expected_components` or `expected_configs` on it. Instead, create the sub-block with the declaration you need using `new()` (see "Reusing a block with `new()`" below). +A composite (`SequentialPipelineBlocks`, `ConditionalPipelineBlocks`, `LoopSequentialPipelineBlocks`) derives its inputs and intermediate outputs from its sub-blocks. They are part of each sub-block's logic, the values its `__call__` reads and writes, so they can't be added or removed on the composite. Don't override `inputs` or `intermediate_outputs` there: to change the default of an input, create the sub-block with the default you need using `new()`. See "Reusing a block with `new()`" below. -The exception is `outputs`, which is what the composite presents as its product. By default it is every intermediate output. Override the property to list only what the step is for, so the docstring shows the product instead of every internal intermediate (Wan-Animate-2's core denoise lists only `segment_frames`). It changes the docstring and `output_names` only: everything stays in the pipeline state, and `pipe(..., output=)` still fetches any intermediate, listed or not. +`outputs` is different: it is what the composite presents as its product. By default it is every intermediate output. Override the property to list only what the step is for, so the docstring shows the product instead of every internal intermediate (most core denoise steps list only `latents`). It changes the docstring and `output_names` only: everything stays in the pipeline state, and `pipe(..., output=)` still fetches any intermediate, listed or not. -When two sub-blocks of a sequence declare different defaults for the same input, the sequence uses the first one; make sure every sub-block has the intended default, with `new()` where it does not. An input a sub-block requires can show as optional on the composite, when an earlier sub-block gives it a default or may produce it; that is expected. +When two sub-blocks of a sequence declare different defaults for the same input, the sequence uses the first one. The best practice is to make sure every sub-block you use has the intended default (use `new()` where it does not). An input a sub-block requires can show as optional on the composite, when an earlier sub-block gives it a default or may produce it; that is expected. ## ComponentSpec patterns @@ -319,6 +319,8 @@ A composite inherits its sub-blocks' `ComponentSpec`s and `ConfigSpec`s. To poin ## Reusing a block with `new()` +A composite's inputs, intermediate outputs, components and configs all derive from its sub-blocks, so you should change the declaration on the sub-block, not on the composite. + A common case: a block does what you need, but one of its declarations does not fit, e.g. the default or description of an input, the repo and loading info of a component, the value of a config. Don't copy or subclass the block for that, create it with `new()`: ```python @@ -339,7 +341,7 @@ class DistilledBlocks(SequentialPipelineBlocks): block_names = ["text_encoder", "set_timesteps", "denoise", "decode"] ``` -`block.new(description=..., inputs=[...], intermediate_outputs=[...], outputs=[...], components=[...], configs=[...])` returns a copy of the block with the listed entries replaced by name. Typical uses: set a preset's default or checkpoint as above, settle a default two sub-blocks disagree on, or reword a description. +`block.new(description=..., inputs=[...], intermediate_outputs=[...], outputs=[...], components=[...], configs=[...])` returns a copy of the block where the entries you pass override the block's own, matched by name. Entries you don't list are unchanged; an entry you do list is replaced whole (gotcha #13). Typical uses: set a preset's default or checkpoint as above, settle a default two sub-blocks disagree on, or reword a description. What `new()` does not do: @@ -376,7 +378,7 @@ What `new()` does not do: 12. **Helpers taking `components`.** Prefer explicit args, for example, `get_xx_prompt_embeds(text_encoder, tokenizer, prompt)` instead of `get_xx_prompt_embeds(components, prompt)`. Helpers and block methods should take only what they use so dependencies are visible and they can run without an assembled pipeline. Leave the full `components` object to `__call__`, which unpacks what its helpers need. See `get_qwen_prompt_embeds` in `qwenimage/encoders.py`. -13. **A `new()` entry that restates less than the block's own.** `block.new(inputs=[InputParam("num_inference_steps", default=10)])` replaces the entry whole, so the type hint and description the block declared are gone. Build it from the template, `InputParam.template("num_inference_steps", default=10)`, or restate everything. +13. **`new()` replaces a whole entry, not a field.** List only the entries you want to change; the others stay as the block declared them. But each entry you list replaces the block's entry completely: `block.new(inputs=[InputParam("num_inference_steps", default=10)])` loses the type hint and description the block had. Build the entry from the template, `InputParam.template("num_inference_steps", default=10)`, or write out every field. ## Conversion checklist diff --git a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py index 34ce5c72b878..ec0cee9a15b2 100644 --- a/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py +++ b/src/diffusers/modular_pipelines/qwenimage/modular_blocks_qwenimage.py @@ -60,8 +60,8 @@ # ==================== -# Kept for backward compatibility, do not copy this for new pipelines: wrapping the text encoder in an auto block makes -# it skippable. Use the text encoder step directly, a user who has embeddings can drop the block +# We normally don't wrap the text encoder in an auto block, since every workflow needs it. +# `QwenImageAutoTextEncoderStep` is the only block that does this, kept for backward compatibility. # auto_docstring class QwenImageAutoTextEncoderStep(AutoPipelineBlocks): """ From ba4872353a5b3fee3c5c453eb26325229babf3ea Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Fri, 20 Nov 2026 09:09:48 +0100 Subject: [PATCH 30/33] [modular tests] `conditional_blocks` takes the sub-blocks directly Co-Authored-By: Claude Fable 5.2 --- .../test_conditional_pipeline_blocks.py | 103 +++++++++++------- 1 file changed, 61 insertions(+), 42 deletions(-) diff --git a/tests/modular_pipelines/test_conditional_pipeline_blocks.py b/tests/modular_pipelines/test_conditional_pipeline_blocks.py index fe207620cbed..36ea861d3472 100644 --- a/tests/modular_pipelines/test_conditional_pipeline_blocks.py +++ b/tests/modular_pipelines/test_conditional_pipeline_blocks.py @@ -29,49 +29,52 @@ def per_branch(name, record, **kwargs): return InputParam(name=name, defaults_by_block=record, **kwargs) -def conditional_blocks(*named_inputs, default_block_name="block-a"): - """ - A conditional block whose sub-blocks declare the given inputs, e.g. - `conditional_blocks(("block-a", [InputParam("x", default=5)]), ("block-b", [InputParam("x", default=5)]))`. - """ - block_classes = [] - for _, block_inputs in named_inputs: +class InputsOnlyBlock(ModularPipelineBlocks): + """A block that declares the given inputs and does nothing.""" - class Block(ModularPipelineBlocks): - model_name = "test" + model_name = "test" + + def __init__(self, inputs): + super().__init__() + self._inputs = inputs - @property - def inputs(self): - return list(self._inputs) + @property + def inputs(self): + return self._inputs + + def __call__(self, components, state): + return components, state - def __call__(self, components, state): - return components, state - Block._inputs = block_inputs # set after the class statement: a class body cannot see the loop variable - block_classes.append(Block) +def conditional_blocks(blocks, default_block_name="block-a"): + """A conditional block over the given `{name: block}` sub-blocks, each triggered by its own name.""" + default = default_block_name class Conditional(ConditionalPipelineBlocks): - pass + block_classes = list(blocks.values()) + block_names = list(blocks) + block_trigger_inputs = list(blocks) + default_block_name = default - Conditional.block_classes = block_classes - Conditional.block_names = [name for name, _ in named_inputs] - Conditional.block_trigger_inputs = Conditional.block_names - Conditional.default_block_name = default_block_name return Conditional() class TestConditionalBlocksInputs: def test_agreeing_defaults_stay_untouched(self): blocks = conditional_blocks( - ("block-a", [InputParam(name="x", default=5)]), - ("block-b", [InputParam(name="x", default=5)]), + { + "block-a": InputsOnlyBlock([InputParam(name="x", default=5)]), + "block-b": InputsOnlyBlock([InputParam(name="x", default=5)]), + } ) assert blocks.inputs == [InputParam(name="x", default=5)] def test_first_occurrence_wins(self): blocks = conditional_blocks( - ("block-a", [InputParam(name="x", default=5, description="from block-a")]), - ("block-b", [InputParam(name="x", default=5, description="from block-b")]), + { + "block-a": InputsOnlyBlock([InputParam(name="x", default=5, description="from block-a")]), + "block-b": InputsOnlyBlock([InputParam(name="x", default=5, description="from block-b")]), + } ) # duplicate inputs keep the first occurrence: block-b's description is dropped assert blocks.inputs == [InputParam(name="x", default=5, description="from block-a")] @@ -79,24 +82,30 @@ def test_first_occurrence_wins(self): def test_none_default_counts_as_disagreement(self): # a None default is a sentinel ("user didn't pass this"), it must not be overridden by a sibling's default blocks = conditional_blocks( - ("block-a", [InputParam(name="x", default=None)]), - ("block-b", [InputParam(name="x", default=5)]), + { + "block-a": InputsOnlyBlock([InputParam(name="x", default=None)]), + "block-b": InputsOnlyBlock([InputParam(name="x", default=5)]), + } ) assert blocks.inputs == [per_branch("x", {"block-a": None, "block-b": 5})] def test_disagreement_records_every_block(self): blocks = conditional_blocks( - ("block-a", [InputParam(name="x", default=1)]), - ("block-b", [InputParam(name="x", default=2)]), - ("block-c", [InputParam(name="x", default=1)]), + { + "block-a": InputsOnlyBlock([InputParam(name="x", default=1)]), + "block-b": InputsOnlyBlock([InputParam(name="x", default=2)]), + "block-c": InputsOnlyBlock([InputParam(name="x", default=1)]), + } ) # any disagreement records every block's default, including the ones that agree with each other assert blocks.inputs == [per_branch("x", {"block-a": 1, "block-b": 2, "block-c": 1})] def test_required_everywhere_stays_required(self): blocks = conditional_blocks( - ("block-a", [InputParam(name="x", required=True)]), - ("block-b", [InputParam(name="x", required=True)]), + { + "block-a": InputsOnlyBlock([InputParam(name="x", required=True)]), + "block-b": InputsOnlyBlock([InputParam(name="x", required=True)]), + } ) assert blocks.inputs == [InputParam(name="x", required=True)] assert blocks.required_inputs == ["x"] @@ -105,8 +114,10 @@ def test_required_everywhere_is_optional_when_the_block_can_be_skipped(self): # no default block: the user can skip the whole block, so nothing it requires is required of the user; the # record still says which sub-blocks require it blocks = conditional_blocks( - ("block-a", [InputParam(name="x", required=True)]), - ("block-b", [InputParam(name="x", required=True)]), + { + "block-a": InputsOnlyBlock([InputParam(name="x", required=True)]), + "block-b": InputsOnlyBlock([InputParam(name="x", required=True)]), + }, default_block_name=None, ) assert blocks.inputs == [per_branch("x", {"block-a": REQUIRED, "block-b": REQUIRED})] @@ -116,15 +127,19 @@ def test_records_a_requirement_other_blocks_do_not_share(self): # block-b does not declare x at all, so x is not required (only when every block requires it): the record is # the only place that says block-a requires it blocks = conditional_blocks( - ("block-a", [InputParam(name="x", required=True)]), - ("block-b", [InputParam(name="y")]), + { + "block-a": InputsOnlyBlock([InputParam(name="x", required=True)]), + "block-b": InputsOnlyBlock([InputParam(name="y")]), + } ) assert blocks.inputs == [per_branch("x", {"block-a": REQUIRED}), InputParam(name="y")] # a requirement against a default is a disagreement like any other blocks = conditional_blocks( - ("block-a", [InputParam(name="x", required=True)]), - ("block-b", [InputParam(name="x", default=124)]), + { + "block-a": InputsOnlyBlock([InputParam(name="x", required=True)]), + "block-b": InputsOnlyBlock([InputParam(name="x", default=124)]), + } ) assert blocks.inputs == [per_branch("x", {"block-a": REQUIRED, "block-b": 124})] @@ -132,8 +147,10 @@ def test_nested_defaults_prefixed(self): # block-b is itself a conditional block whose branches already disagree, so its param carries # defaults_by_block; merging at the outer level prefixes those entries with the sub-block name blocks = conditional_blocks( - ("block-a", [InputParam(name="x", default=3)]), - ("block-b", [per_branch("x", {"branch-a": 1, "branch-b": 2})]), + { + "block-a": InputsOnlyBlock([InputParam(name="x", default=3)]), + "block-b": InputsOnlyBlock([per_branch("x", {"branch-a": 1, "branch-b": 2})]), + } ) assert blocks.inputs == [per_branch("x", {"block-a": 3, "block-b.branch-a": 1, "block-b.branch-b": 2})] @@ -141,8 +158,10 @@ def test_nested_requirements_stay_optional(self): # both sub-blocks are conditional blocks with one branch requiring x and another silent on it, so each is # optional with a record; the records agree (REQUIRED, REQUIRED) but x is still not required of the user blocks = conditional_blocks( - ("block-a", [per_branch("x", {"branch-a": REQUIRED})]), - ("block-b", [per_branch("x", {"branch-b": REQUIRED})]), + { + "block-a": InputsOnlyBlock([per_branch("x", {"branch-a": REQUIRED})]), + "block-b": InputsOnlyBlock([per_branch("x", {"branch-b": REQUIRED})]), + } ) assert blocks.inputs == [per_branch("x", {"block-a.branch-a": REQUIRED, "block-b.branch-b": REQUIRED})] assert blocks.required_inputs == [] From 1f73330d4114b50cc5b4b932df5146a3f2524e5b Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Fri, 20 Nov 2026 09:13:09 +0100 Subject: [PATCH 31/33] [modular tests] inline the `per_branch` test helper Co-Authored-By: Claude Fable 5.2 --- .../test_conditional_pipeline_blocks.py | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/tests/modular_pipelines/test_conditional_pipeline_blocks.py b/tests/modular_pipelines/test_conditional_pipeline_blocks.py index 36ea861d3472..061bb53128c9 100644 --- a/tests/modular_pipelines/test_conditional_pipeline_blocks.py +++ b/tests/modular_pipelines/test_conditional_pipeline_blocks.py @@ -24,11 +24,6 @@ from diffusers.modular_pipelines.modular_pipeline_utils import REQUIRED -def per_branch(name, record, **kwargs): - """An `InputParam` carrying a per-branch record, the way `ConditionalPipelineBlocks` builds it.""" - return InputParam(name=name, defaults_by_block=record, **kwargs) - - class InputsOnlyBlock(ModularPipelineBlocks): """A block that declares the given inputs and does nothing.""" @@ -87,7 +82,7 @@ def test_none_default_counts_as_disagreement(self): "block-b": InputsOnlyBlock([InputParam(name="x", default=5)]), } ) - assert blocks.inputs == [per_branch("x", {"block-a": None, "block-b": 5})] + assert blocks.inputs == [InputParam(name="x", defaults_by_block={"block-a": None, "block-b": 5})] def test_disagreement_records_every_block(self): blocks = conditional_blocks( @@ -98,7 +93,7 @@ def test_disagreement_records_every_block(self): } ) # any disagreement records every block's default, including the ones that agree with each other - assert blocks.inputs == [per_branch("x", {"block-a": 1, "block-b": 2, "block-c": 1})] + assert blocks.inputs == [InputParam(name="x", defaults_by_block={"block-a": 1, "block-b": 2, "block-c": 1})] def test_required_everywhere_stays_required(self): blocks = conditional_blocks( @@ -120,7 +115,7 @@ def test_required_everywhere_is_optional_when_the_block_can_be_skipped(self): }, default_block_name=None, ) - assert blocks.inputs == [per_branch("x", {"block-a": REQUIRED, "block-b": REQUIRED})] + assert blocks.inputs == [InputParam(name="x", defaults_by_block={"block-a": REQUIRED, "block-b": REQUIRED})] assert blocks.required_inputs == [] def test_records_a_requirement_other_blocks_do_not_share(self): @@ -132,7 +127,7 @@ def test_records_a_requirement_other_blocks_do_not_share(self): "block-b": InputsOnlyBlock([InputParam(name="y")]), } ) - assert blocks.inputs == [per_branch("x", {"block-a": REQUIRED}), InputParam(name="y")] + assert blocks.inputs == [InputParam(name="x", defaults_by_block={"block-a": REQUIRED}), InputParam(name="y")] # a requirement against a default is a disagreement like any other blocks = conditional_blocks( @@ -141,7 +136,7 @@ def test_records_a_requirement_other_blocks_do_not_share(self): "block-b": InputsOnlyBlock([InputParam(name="x", default=124)]), } ) - assert blocks.inputs == [per_branch("x", {"block-a": REQUIRED, "block-b": 124})] + assert blocks.inputs == [InputParam(name="x", defaults_by_block={"block-a": REQUIRED, "block-b": 124})] def test_nested_defaults_prefixed(self): # block-b is itself a conditional block whose branches already disagree, so its param carries @@ -149,21 +144,25 @@ def test_nested_defaults_prefixed(self): blocks = conditional_blocks( { "block-a": InputsOnlyBlock([InputParam(name="x", default=3)]), - "block-b": InputsOnlyBlock([per_branch("x", {"branch-a": 1, "branch-b": 2})]), + "block-b": InputsOnlyBlock([InputParam(name="x", defaults_by_block={"branch-a": 1, "branch-b": 2})]), } ) - assert blocks.inputs == [per_branch("x", {"block-a": 3, "block-b.branch-a": 1, "block-b.branch-b": 2})] + assert blocks.inputs == [ + InputParam(name="x", defaults_by_block={"block-a": 3, "block-b.branch-a": 1, "block-b.branch-b": 2}) + ] def test_nested_requirements_stay_optional(self): # both sub-blocks are conditional blocks with one branch requiring x and another silent on it, so each is # optional with a record; the records agree (REQUIRED, REQUIRED) but x is still not required of the user blocks = conditional_blocks( { - "block-a": InputsOnlyBlock([per_branch("x", {"branch-a": REQUIRED})]), - "block-b": InputsOnlyBlock([per_branch("x", {"branch-b": REQUIRED})]), + "block-a": InputsOnlyBlock([InputParam(name="x", defaults_by_block={"branch-a": REQUIRED})]), + "block-b": InputsOnlyBlock([InputParam(name="x", defaults_by_block={"branch-b": REQUIRED})]), } ) - assert blocks.inputs == [per_branch("x", {"block-a.branch-a": REQUIRED, "block-b.branch-b": REQUIRED})] + assert blocks.inputs == [ + InputParam(name="x", defaults_by_block={"block-a.branch-a": REQUIRED, "block-b.branch-b": REQUIRED}) + ] assert blocks.required_inputs == [] @@ -440,7 +439,9 @@ def test_conflicting_defaults_merge_to_none(self): # and the per-block defaults are recorded in defaults_by_block strength_params = [p for p in AutoImageBlocks().inputs if p.name == "strength"] assert len(strength_params) == 1 - assert strength_params[0] == per_branch("strength", {"inpaint": 0.9999, "img2img": 0.3}, type_hint=float) + assert strength_params[0] == InputParam( + name="strength", type_hint=float, defaults_by_block={"inpaint": 0.9999, "img2img": 0.3} + ) # the docstring does not pick one of the per-block defaults doc = " ".join(AutoImageBlocks().doc.split()) @@ -471,8 +472,10 @@ def test_standalone_branch_keeps_default(self): def test_nested_defaults_prefixed_with_sub_block_name(self): strength_params = [p for p in NestedImageBlocks().inputs if p.name == "strength"] assert len(strength_params) == 1 - assert strength_params[0] == per_branch( - "strength", {"refine": 0.9999, "image.inpaint": 0.9999, "image.img2img": 0.3}, type_hint=float + assert strength_params[0] == InputParam( + name="strength", + type_hint=float, + defaults_by_block={"refine": 0.9999, "image.inpaint": 0.9999, "image.img2img": 0.3}, ) doc = " ".join(NestedImageBlocks().doc.split()) From 6c3f206e7cab451ddf6bdd4434c80fedbd1e3771 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Fri, 20 Nov 2026 09:31:06 +0100 Subject: [PATCH 32/33] [modular tests] build real nested blocks in `test_nested_requirements_stay_optional` Co-Authored-By: Claude Fable 5.2 --- .../test_conditional_pipeline_blocks.py | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/modular_pipelines/test_conditional_pipeline_blocks.py b/tests/modular_pipelines/test_conditional_pipeline_blocks.py index 061bb53128c9..4e02d8e275d8 100644 --- a/tests/modular_pipelines/test_conditional_pipeline_blocks.py +++ b/tests/modular_pipelines/test_conditional_pipeline_blocks.py @@ -152,17 +152,28 @@ def test_nested_defaults_prefixed(self): ] def test_nested_requirements_stay_optional(self): - # both sub-blocks are conditional blocks with one branch requiring x and another silent on it, so each is - # optional with a record; the records agree (REQUIRED, REQUIRED) but x is still not required of the user - blocks = conditional_blocks( + # each sub-block is a conditional block where one branch requires x and the other does not use it, so x is + # not required of the user even though every record says REQUIRED + block_a = conditional_blocks( { - "block-a": InputsOnlyBlock([InputParam(name="x", defaults_by_block={"branch-a": REQUIRED})]), - "block-b": InputsOnlyBlock([InputParam(name="x", defaults_by_block={"branch-b": REQUIRED})]), - } + "branch-a": InputsOnlyBlock([InputParam(name="x", required=True)]), + "other": InputsOnlyBlock([InputParam(name="y")]), + }, + default_block_name="branch-a", + ) + block_b = conditional_blocks( + { + "branch-b": InputsOnlyBlock([InputParam(name="x", required=True)]), + "other": InputsOnlyBlock([InputParam(name="y")]), + }, + default_block_name="branch-b", + ) + blocks = conditional_blocks({"block-a": block_a, "block-b": block_b}) + + x = blocks.inputs[0] + assert x == InputParam( + name="x", defaults_by_block={"block-a.branch-a": REQUIRED, "block-b.branch-b": REQUIRED} ) - assert blocks.inputs == [ - InputParam(name="x", defaults_by_block={"block-a.branch-a": REQUIRED, "block-b.branch-b": REQUIRED}) - ] assert blocks.required_inputs == [] From 3dfd64866a5234b0145abe49648bace46b8d0278 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Fri, 20 Nov 2026 09:39:41 +0100 Subject: [PATCH 33/33] [modular tests] plain block classes for the optional-output tests; `from_blocks_dict` signature back on one line Co-Authored-By: Claude Fable 5.2 --- .../modular_pipelines/modular_pipeline.py | 5 +--- .../test_conditional_pipeline_blocks.py | 26 ++++++++++--------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 2409006595e7..aba0be56d036 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -1827,10 +1827,7 @@ def __init__(self): ) @classmethod - def from_blocks_dict( - cls, - blocks_dict: dict[str, Any], - ) -> "LoopSequentialPipelineBlocks": + def from_blocks_dict(cls, blocks_dict: dict[str, Any]) -> "LoopSequentialPipelineBlocks": """ Creates a LoopSequentialPipelineBlocks instance from a dictionary of blocks. diff --git a/tests/modular_pipelines/test_conditional_pipeline_blocks.py b/tests/modular_pipelines/test_conditional_pipeline_blocks.py index 4e02d8e275d8..446982599061 100644 --- a/tests/modular_pipelines/test_conditional_pipeline_blocks.py +++ b/tests/modular_pipelines/test_conditional_pipeline_blocks.py @@ -560,21 +560,23 @@ def __call__(self, components, state): return components, state -class TestConditionalBlocksOptionalOutputs: - def _blocks(self, default_block_name="text2img"): - class Blocks(ConditionalPipelineBlocks): - block_classes = [ProducesLatentsBlock, ProducesLatentsAndImageLatentsBlock] - block_names = ["img2img", "text2img"] - block_trigger_inputs = ["image"] +class LatentsBlocks(ConditionalPipelineBlocks): + block_classes = [ProducesLatentsBlock, ProducesLatentsAndImageLatentsBlock] + block_names = ["img2img", "text2img"] + block_trigger_inputs = ["image"] + default_block_name = "text2img" + + def select_block(self, image=None): + return "img2img" if image is not None else None - def select_block(self, image=None): - return "img2img" if image is not None else None - Blocks.default_block_name = default_block_name - return Blocks() +class SkippableLatentsBlocks(LatentsBlocks): + default_block_name = None # no default; block can be skipped + +class TestConditionalBlocksOptionalOutputs: def test_output_is_always_produced_only_when_every_branch_produces_it(self): - blocks = self._blocks() + blocks = LatentsBlocks() # `latents` comes out of both branches, `image_latents` out of `text2img` only assert {out.name: out._maybe for out in blocks.intermediate_outputs} == { "latents": False, @@ -584,7 +586,7 @@ def test_output_is_always_produced_only_when_every_branch_produces_it(self): assert all(not out._maybe for out in ProducesLatentsAndImageLatentsBlock().intermediate_outputs) def test_every_output_of_a_skippable_block_is_a_maybe(self): - blocks = self._blocks(default_block_name=None) + blocks = SkippableLatentsBlocks() assert {out.name: out._maybe for out in blocks.intermediate_outputs} == { "latents": True, "image_latents": True,