From 887c5f20e04ed7550afbeb8f4268652fd4da2408 Mon Sep 17 00:00:00 2001 From: sdiazlor Date: Fri, 20 Nov 2026 14:18:22 +0100 Subject: [PATCH 1/2] Rename P-H3-Image to P-Image-OSS Move the autoencoder, condition embedder, modular pipeline, docs and tests to the p_image_oss name. Keep the old p_h3_image modules as deprecated aliases. Co-Authored-By: Claude Opus 5.5 --- docs/source/en/_redirects.yml | 2 + docs/source/en/_toctree.yml | 8 +- ..._image.md => autoencoderkl_p_image_oss.md} | 12 +- .../{p_h3_image.md => p_image_oss.md} | 33 +- src/diffusers/__init__.py | 8 + src/diffusers/models/__init__.py | 9 +- src/diffusers/models/autoencoders/__init__.py | 2 +- .../autoencoders/autoencoder_kl_p_h3_image.py | 798 +---------------- .../autoencoder_kl_p_image_oss.py | 814 ++++++++++++++++++ .../models/condition_embedders/__init__.py | 2 +- .../condition_embedder_p_h3_image.py | 65 +- .../condition_embedder_p_image_oss.py | 81 ++ src/diffusers/modular_pipelines/__init__.py | 8 +- .../modular_pipelines/modular_pipeline.py | 4 +- .../modular_pipelines/p_h3_image/__init__.py | 61 +- .../p_h3_image/modular_blocks_p_h3_image.py | 456 +--------- .../p_h3_image/modular_pipeline.py | 113 +-- .../modular_pipelines/p_image_oss/__init__.py | 47 + .../before_denoise.py | 54 +- .../before_encoder.py | 10 +- .../{p_h3_image => p_image_oss}/decoders.py | 24 +- .../{p_h3_image => p_image_oss}/denoise.py | 38 +- .../{p_h3_image => p_image_oss}/encoders.py | 52 +- .../p_image_oss/modular_blocks_p_image_oss.py | 472 ++++++++++ .../p_image_oss/modular_pipeline.py | 127 +++ .../prompt_upsampler.py | 44 +- src/diffusers/utils/dummy_pt_objects.py | 30 + .../dummy_torch_and_transformers_objects.py | 30 + ...test_models_autoencoder_kl_p_image_oss.py} | 28 +- .../{p_h3_image => p_image_oss}/__init__.py | 0 .../test_modular_pipeline_p_image_oss.py} | 98 +-- 31 files changed, 1850 insertions(+), 1680 deletions(-) rename docs/source/en/api/models/{autoencoderkl_p_h3_image.md => autoencoderkl_p_image_oss.md} (72%) rename docs/source/en/api/pipelines/{p_h3_image.md => p_image_oss.md} (74%) create mode 100644 src/diffusers/models/autoencoders/autoencoder_kl_p_image_oss.py create mode 100644 src/diffusers/models/condition_embedders/condition_embedder_p_image_oss.py create mode 100644 src/diffusers/modular_pipelines/p_image_oss/__init__.py rename src/diffusers/modular_pipelines/{p_h3_image => p_image_oss}/before_denoise.py (96%) rename src/diffusers/modular_pipelines/{p_h3_image => p_image_oss}/before_encoder.py (96%) rename src/diffusers/modular_pipelines/{p_h3_image => p_image_oss}/decoders.py (91%) rename src/diffusers/modular_pipelines/{p_h3_image => p_image_oss}/denoise.py (88%) rename src/diffusers/modular_pipelines/{p_h3_image => p_image_oss}/encoders.py (87%) create mode 100644 src/diffusers/modular_pipelines/p_image_oss/modular_blocks_p_image_oss.py create mode 100644 src/diffusers/modular_pipelines/p_image_oss/modular_pipeline.py rename src/diffusers/modular_pipelines/{p_h3_image => p_image_oss}/prompt_upsampler.py (88%) rename tests/models/autoencoders/{test_models_autoencoder_kl_p_h3_image.py => test_models_autoencoder_kl_p_image_oss.py} (81%) rename tests/modular_pipelines/{p_h3_image => p_image_oss}/__init__.py (100%) rename tests/modular_pipelines/{p_h3_image/test_modular_pipeline_p_h3_image.py => p_image_oss/test_modular_pipeline_p_image_oss.py} (85%) diff --git a/docs/source/en/_redirects.yml b/docs/source/en/_redirects.yml index 832b6f05fa54..978b99c2f5fa 100644 --- a/docs/source/en/_redirects.yml +++ b/docs/source/en/_redirects.yml @@ -1,2 +1,4 @@ # old_local: new_local stable_diffusion: optimization/overview +api/pipelines/p_h3_image: api/pipelines/p_image_oss +api/models/autoencoderkl_p_h3_image: api/models/autoencoderkl_p_image_oss diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index e1a4a9247894..21fc98873713 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -515,8 +515,8 @@ title: AutoencoderKLMiniMaxH3Image - local: api/models/autoencoderkl_mochi title: AutoencoderKLMochi - - local: api/models/autoencoderkl_p_h3_image - title: AutoencoderKLPH3Image + - local: api/models/autoencoderkl_p_image_oss + title: AutoencoderKLPImageOSS - local: api/models/autoencoderkl_qwenimage title: AutoencoderKLQwenImage - local: api/models/autoencoderkl_qwenimage21 @@ -665,8 +665,8 @@ title: OmniGen - local: api/pipelines/ovis_image title: Ovis-Image - - local: api/pipelines/p_h3_image - title: P-H3-Image + - local: api/pipelines/p_image_oss + title: P-Image-OSS - local: api/pipelines/pag title: PAG - local: api/pipelines/pixart diff --git a/docs/source/en/api/models/autoencoderkl_p_h3_image.md b/docs/source/en/api/models/autoencoderkl_p_image_oss.md similarity index 72% rename from docs/source/en/api/models/autoencoderkl_p_h3_image.md rename to docs/source/en/api/models/autoencoderkl_p_image_oss.md index 5cecb83d9b73..418b720a8571 100644 --- a/docs/source/en/api/models/autoencoderkl_p_h3_image.md +++ b/docs/source/en/api/models/autoencoderkl_p_image_oss.md @@ -9,9 +9,9 @@ Unless required by applicable law or agreed to in writing, software distributed an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -# AutoencoderKLPH3Image +# AutoencoderKLPImageOSS -The image VAE of [P-H3-Image](https://huggingface.co/PrunaAI/P-H3-Image) by Pruna AI. It uses the encoder and latent space of MiniMax-H3's [video VAE](./autoencoderkl_minimax_h3), with a ViT decoder that decodes a single latent frame into an image. +The image VAE of [P-Image-OSS](https://huggingface.co/PrunaAI/P-Image-OSS) by Pruna AI. It uses the encoder and latent space of MiniMax-H3's [video VAE](./autoencoderkl_minimax_h3), with a ViT decoder that decodes a single latent frame into an image. It follows the conventions of [`AutoencoderKLMiniMaxH3`]: @@ -22,18 +22,18 @@ It follows the conventions of [`AutoencoderKLMiniMaxH3`]: ```python import torch -from diffusers import AutoencoderKLPH3Image +from diffusers import AutoencoderKLPImageOSS -vae = AutoencoderKLPH3Image.from_pretrained("PrunaAI/P-H3-Image", subfolder="vae", dtype=torch.bfloat16).to( +vae = AutoencoderKLPImageOSS.from_pretrained("PrunaAI/P-Image-OSS", subfolder="vae", dtype=torch.bfloat16).to( "cuda" ) # or "mps", "xpu", "cpu" ``` The encoder and the decoder's output projection stay in float32 whatever `dtype` the model is loaded in. -## AutoencoderKLPH3Image +## AutoencoderKLPImageOSS -[[autodoc]] AutoencoderKLPH3Image +[[autodoc]] AutoencoderKLPImageOSS - encode - decode - all diff --git a/docs/source/en/api/pipelines/p_h3_image.md b/docs/source/en/api/pipelines/p_image_oss.md similarity index 74% rename from docs/source/en/api/pipelines/p_h3_image.md rename to docs/source/en/api/pipelines/p_image_oss.md index 6740298762b8..496683a539ae 100644 --- a/docs/source/en/api/pipelines/p_h3_image.md +++ b/docs/source/en/api/pipelines/p_image_oss.md @@ -9,9 +9,12 @@ Unless required by applicable law or agreed to in writing, software distributed an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -# P-H3-Image +# P-Image-OSS -P-H3-Image is an image model by Pruna AI based on [MiniMax-H3](./minimax_h3), for text-to-image generation and instruction-based image editing. You can find the checkpoint at [PrunaAI/P-H3-Image](https://huggingface.co/PrunaAI/P-H3-Image). +P-Image-OSS is an image model by Pruna AI based on [MiniMax-H3](./minimax_h3), for text-to-image generation and instruction-based image editing. You can find the checkpoint at [PrunaAI/P-Image-OSS](https://huggingface.co/PrunaAI/P-Image-OSS). + +> [!TIP] +> P-Image-OSS was previously released as P-H3-Image. The former class names (`PH3ImageModularPipeline`, `PH3ImageBlocks`, `PH3ImageTextConnector`, and `AutoencoderKLPH3Image`) are kept as deprecated aliases of the new ones. ```py import torch @@ -20,7 +23,7 @@ from diffusers import ComponentsManager, ModularPipeline manager = ComponentsManager() manager.enable_auto_cpu_offload(device="cuda") # or "mps", "xpu" -pipe = ModularPipeline.from_pretrained("PrunaAI/P-H3-Image", components_manager=manager) +pipe = ModularPipeline.from_pretrained("PrunaAI/P-Image-OSS", components_manager=manager) pipe.load_components(dtype=torch.bfloat16) prompt = ( @@ -39,8 +42,8 @@ Every request goes through the [prompt upsampler](#prompt-upsampling) first. Wit |---|---| | `transformer` | [`MiniMaxH3Transformer3DModel`] | | `text_encoder` | `Qwen3VLForConditionalGeneration` (Qwen3-VL-4B) | -| `text_connector` | [`PH3ImageTextConnector`] | -| `vae` | [`AutoencoderKLPH3Image`] | +| `text_connector` | [`PImageOSSTextConnector`] | +| `vae` | [`AutoencoderKLPImageOSS`] | | `tokenizer`, `processor` | The text encoder's tokenizer, and a `Qwen3VLProcessor` with MiniMax-H3's image processor | | `scheduler`, `audio_scheduler` | [`MiniMaxH3Scheduler`] | | `prompt_upsampler`, `prompt_upsampler_processor` | `Qwen3_5ForConditionalGeneration` (based on Qwen3.5-9B) and its processor | @@ -54,7 +57,7 @@ The workflow is selected on the `image` input: ## Prompt upsampling -P-H3-Image is meant to render full prompts rather than short requests, so the pipeline runs the prompt upsampler first on every request. It thinks about the request and then writes the prompt: a detailed text-to-image prompt for the aspect ratio of `height` and `width`, or, when editing, a brief of what to keep and what to change in the images. The prompt it wrote comes back as the `prompt` output. +P-Image-OSS is meant to render full prompts rather than short requests, so the pipeline runs the prompt upsampler first on every request. It thinks about the request and then writes the prompt: a detailed text-to-image prompt for the aspect ratio of `height` and `width`, or, when editing, a brief of what to keep and what to change in the images. The prompt it wrote comes back as the `prompt` output. ```py out = pipe(prompt=prompt, height=3360, width=2496, output=["images", "prompt"]) @@ -69,14 +72,14 @@ To render `prompt` as given, pop the step from the blocks. The popped block also manager = ComponentsManager() manager.enable_auto_cpu_offload(device="cuda") -blocks = ModularPipeline.from_pretrained("PrunaAI/P-H3-Image").blocks +blocks = ModularPipeline.from_pretrained("PrunaAI/P-Image-OSS").blocks upsampler_block = blocks.sub_blocks.pop("prompt_upsampler") -upsampler = upsampler_block.init_pipeline("PrunaAI/P-H3-Image", components_manager=manager) +upsampler = upsampler_block.init_pipeline("PrunaAI/P-Image-OSS", components_manager=manager) upsampler.load_components(dtype=torch.bfloat16) upsampled = upsampler(prompt=prompt, height=3360, width=2496, output="prompt") -pipe = blocks.init_pipeline("PrunaAI/P-H3-Image", components_manager=manager) +pipe = blocks.init_pipeline("PrunaAI/P-Image-OSS", components_manager=manager) pipe.load_components(dtype=torch.bfloat16) image = pipe(prompt=upsampled, height=3360, width=2496, output="images")[0] ``` @@ -88,14 +91,14 @@ Without the step, an edit prompt in either form the upsampler writes, the six-se - The 8 steps run over the checkpoint's `sample_sigmas`. Pass `sigmas` to use another grid. - The model is guidance-distilled, so there is no guider, `negative_prompt` or `guidance_scale`. -## PH3ImageModularPipeline +## PImageOSSModularPipeline -[[autodoc]] PH3ImageModularPipeline +[[autodoc]] PImageOSSModularPipeline -## PH3ImageBlocks +## PImageOSSBlocks -[[autodoc]] PH3ImageBlocks +[[autodoc]] PImageOSSBlocks -## PH3ImageTextConnector +## PImageOSSTextConnector -[[autodoc]] PH3ImageTextConnector +[[autodoc]] PImageOSSTextConnector diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index 1435abc234cf..b5f3134e42da 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -262,6 +262,7 @@ "AutoencoderKLMiniMaxH3Image", "AutoencoderKLMochi", "AutoencoderKLPH3Image", + "AutoencoderKLPImageOSS", "AutoencoderKLQwenImage", "AutoencoderKLQwenImage21", "AutoencoderKLTemporalDecoder", @@ -358,6 +359,7 @@ "PAN2Transformer3DModel", "ParallelConfig", "PH3ImageTextConnector", + "PImageOSSTextConnector", "PixArtTransformer2DModel", "PriorTransformer", "PRXTransformer2DModel", @@ -611,6 +613,8 @@ "PAN2SafetyChecker", "PH3ImageBlocks", "PH3ImageModularPipeline", + "PImageOSSBlocks", + "PImageOSSModularPipeline", "PikaH3FlexBlocks", "PikaH3FlexModularPipeline", "QwenImage21AutoBlocks", @@ -1180,6 +1184,7 @@ AutoencoderKLMiniMaxH3Image, AutoencoderKLMochi, AutoencoderKLPH3Image, + AutoencoderKLPImageOSS, AutoencoderKLQwenImage, AutoencoderKLQwenImage21, AutoencoderKLTemporalDecoder, @@ -1276,6 +1281,7 @@ PAN2Transformer3DModel, ParallelConfig, PH3ImageTextConnector, + PImageOSSTextConnector, PixArtTransformer2DModel, PriorTransformer, PRXTransformer2DModel, @@ -1509,6 +1515,8 @@ PH3ImageModularPipeline, PikaH3FlexBlocks, PikaH3FlexModularPipeline, + PImageOSSBlocks, + PImageOSSModularPipeline, QwenImage21AutoBlocks, QwenImage21ModularPipeline, QwenImage21TurboAutoBlocks, diff --git a/src/diffusers/models/__init__.py b/src/diffusers/models/__init__.py index 409933b0304d..8fb161f13b19 100755 --- a/src/diffusers/models/__init__.py +++ b/src/diffusers/models/__init__.py @@ -56,7 +56,7 @@ _import_structure["autoencoders.autoencoder_kl_minimax_h3_audio"] = ["AutoencoderKLMiniMaxH3Audio"] _import_structure["autoencoders.autoencoder_kl_minimax_h3_image"] = ["AutoencoderKLMiniMaxH3Image"] _import_structure["autoencoders.autoencoder_kl_mochi"] = ["AutoencoderKLMochi"] - _import_structure["autoencoders.autoencoder_kl_p_h3_image"] = ["AutoencoderKLPH3Image"] + _import_structure["autoencoders.autoencoder_kl_p_image_oss"] = ["AutoencoderKLPH3Image", "AutoencoderKLPImageOSS"] _import_structure["autoencoders.autoencoder_kl_qwenimage"] = ["AutoencoderKLQwenImage"] _import_structure["autoencoders.autoencoder_kl_qwenimage21"] = ["AutoencoderKLQwenImage21"] _import_structure["autoencoders.autoencoder_kl_temporal_decoder"] = ["AutoencoderKLTemporalDecoder"] @@ -83,7 +83,10 @@ _import_structure["condition_embedders.condition_embedder_gligen"] = ["CLIPImageProjection"] _import_structure["condition_embedders.condition_embedder_ltx2"] = ["LTX2TextConnectors"] _import_structure["condition_embedders.condition_embedder_minimax_music3"] = ["MiniMaxMusic3ConditionEncoder"] - _import_structure["condition_embedders.condition_embedder_p_h3_image"] = ["PH3ImageTextConnector"] + _import_structure["condition_embedders.condition_embedder_p_image_oss"] = [ + "PH3ImageTextConnector", + "PImageOSSTextConnector", + ] _import_structure["condition_embedders.condition_embedder_stable_audio"] = ["StableAudioProjectionModel"] _import_structure["condition_embedders.condition_embedder_stable_audio_3"] = ["StableAudio3DurationEmbedder"] _import_structure["condition_embedders.condition_embedder_stable_unclip"] = ["StableUnCLIPImageNormalizer"] @@ -236,6 +239,7 @@ AutoencoderKLMiniMaxH3Image, AutoencoderKLMochi, AutoencoderKLPH3Image, + AutoencoderKLPImageOSS, AutoencoderKLQwenImage, AutoencoderKLQwenImage21, AutoencoderKLTemporalDecoder, @@ -267,6 +271,7 @@ LTX2TextConnectors, MiniMaxMusic3ConditionEncoder, PH3ImageTextConnector, + PImageOSSTextConnector, ReduxImageEncoder, StableAudio3DurationEmbedder, StableAudioProjectionModel, diff --git a/src/diffusers/models/autoencoders/__init__.py b/src/diffusers/models/autoencoders/__init__.py index 08e808d06363..fc9bdc32995f 100644 --- a/src/diffusers/models/autoencoders/__init__.py +++ b/src/diffusers/models/autoencoders/__init__.py @@ -23,7 +23,7 @@ from .autoencoder_kl_minimax_h3_audio import AutoencoderKLMiniMaxH3Audio from .autoencoder_kl_minimax_h3_image import AutoencoderKLMiniMaxH3Image from .autoencoder_kl_mochi import AutoencoderKLMochi -from .autoencoder_kl_p_h3_image import AutoencoderKLPH3Image +from .autoencoder_kl_p_image_oss import AutoencoderKLPH3Image, AutoencoderKLPImageOSS from .autoencoder_kl_qwenimage import AutoencoderKLQwenImage from .autoencoder_kl_qwenimage21 import AutoencoderKLQwenImage21 from .autoencoder_kl_temporal_decoder import AutoencoderKLTemporalDecoder diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_p_h3_image.py b/src/diffusers/models/autoencoders/autoencoder_kl_p_h3_image.py index 87ac55fc872f..d130d00f238d 100644 --- a/src/diffusers/models/autoencoders/autoencoder_kl_p_h3_image.py +++ b/src/diffusers/models/autoencoders/autoencoder_kl_p_h3_image.py @@ -11,800 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import math - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from ...configuration_utils import ConfigMixin, register_to_config -from ...utils.accelerate_utils import apply_forward_hook -from ..attention import AttentionMixin, AttentionModuleMixin, FeedForward -from ..attention_dispatch import dispatch_attention_fn -from ..modeling_outputs import AutoencoderKLOutput -from ..modeling_utils import ModelMixin, get_parameter_dtype -from .vae import AutoencoderMixin, DecoderOutput, DiagonalGaussianDistribution - - -# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoCausalConv3d with MiniMaxH3Video->PH3Image -class PH3ImageCausalConv3d(nn.Conv3d): - r""" - 3D convolution used throughout the MiniMax-H3 video encoder. - - Spatial padding is symmetric and uses `spatial_padding_mode` (`"reflect"` in the released checkpoint); temporal - padding is causal, i.e. `kernel_size_t - 1` zero frames are prepended and nothing is appended. - """ - - def __init__( - self, - in_channels: int, - out_channels: int, - kernel_size: int | tuple[int, int, int], - stride: int | tuple[int, int, int] = 1, - spatial_padding: int = 0, - temporal_padding: int = 0, - spatial_padding_mode: str = "reflect", - ) -> None: - super().__init__(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=0) - self.spatial_padding = spatial_padding - self.temporal_padding = temporal_padding - self.spatial_padding_mode = spatial_padding_mode - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - if self.spatial_padding > 0: - padding = self.spatial_padding - hidden_states = F.pad( - hidden_states, (padding, padding, padding, padding, 0, 0), mode=self.spatial_padding_mode - ) - if self.temporal_padding > 0: - hidden_states = F.pad(hidden_states, (0, 0, 0, 0, self.temporal_padding, 0), mode="constant") - return F.conv3d(hidden_states, self.weight, self.bias, stride=self.stride, padding=0, dilation=self.dilation) - - -# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoGroupNorm with MiniMaxH3Video->PH3Image -class PH3ImageGroupNorm(nn.GroupNorm): - r""" - Group normalization applied to each latent frame in isolation (`use_t_isolated_gn` in the original config): the - temporal axis is folded into the batch axis so statistics never mix across frames. - """ - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - batch_size, num_channels, num_frames, height, width = hidden_states.shape - hidden_states = hidden_states.permute(0, 2, 1, 3, 4).contiguous() - hidden_states = hidden_states.view(batch_size * num_frames, num_channels, 1, height, width) - hidden_states = super().forward(hidden_states) - hidden_states = hidden_states.view(batch_size, num_frames, num_channels, height, width) - return hidden_states.permute(0, 2, 1, 3, 4).contiguous() - - -# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoResnetBlock3d with MiniMaxH3Video->PH3Image -class PH3ImageResnetBlock3d(nn.Module): - def __init__( - self, - in_channels: int, - out_channels: int, - norm_num_groups: int = 32, - norm_eps: float = 1e-6, - spatial_padding_mode: str = "reflect", - ) -> None: - super().__init__() - self.in_channels = in_channels - self.out_channels = out_channels - - self.norm1 = PH3ImageGroupNorm(norm_num_groups, in_channels, eps=norm_eps, affine=True) - self.conv1 = PH3ImageCausalConv3d( - in_channels, - out_channels, - kernel_size=3, - spatial_padding=1, - temporal_padding=2, - spatial_padding_mode=spatial_padding_mode, - ) - self.norm2 = PH3ImageGroupNorm(norm_num_groups, out_channels, eps=norm_eps, affine=True) - self.conv2 = PH3ImageCausalConv3d( - out_channels, - out_channels, - kernel_size=3, - spatial_padding=1, - temporal_padding=2, - spatial_padding_mode=spatial_padding_mode, - ) - self.conv_shortcut = None - if in_channels != out_channels: - self.conv_shortcut = PH3ImageCausalConv3d(in_channels, out_channels, kernel_size=1) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - residual = hidden_states - hidden_states = F.silu(self.norm1(hidden_states)) - hidden_states = self.conv1(hidden_states) - hidden_states = F.silu(self.norm2(hidden_states)) - hidden_states = self.conv2(hidden_states) - if self.conv_shortcut is not None: - residual = self.conv_shortcut(residual) - return residual + hidden_states - - -# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoDownsample3d with MiniMaxH3Video->PH3Image -class PH3ImageDownsample3d(nn.Module): - r""" - Strided 3x3x3 downsampling convolution. A spatial stride of 2 is preceded by an asymmetric bottom/right pad of 1 - (the convolution itself carries no spatial padding), so the output is exactly `ceil(size / 2)`. - """ - - def __init__( - self, - in_channels: int, - out_channels: int, - temporal_stride: int = 1, - spatial_stride: int = 2, - spatial_padding_mode: str = "reflect", - ) -> None: - super().__init__() - self.spatial_stride = spatial_stride - self.spatial_padding_mode = spatial_padding_mode - self.conv = PH3ImageCausalConv3d( - in_channels, - out_channels, - kernel_size=3, - stride=(temporal_stride, spatial_stride, spatial_stride), - spatial_padding=0, - temporal_padding=2, - spatial_padding_mode=spatial_padding_mode, - ) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - if self.spatial_stride == 2: - hidden_states = F.pad(hidden_states, (0, 1, 0, 1, 0, 0), mode=self.spatial_padding_mode) - return self.conv(hidden_states) - - -# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoDownBlock3d with MiniMaxH3Video->PH3Image -class PH3ImageDownBlock3d(nn.Module): - def __init__( - self, - in_channels: int, - out_channels: int, - num_layers: int, - temporal_downsample_factor: int, - spatial_downsample_factor: int, - norm_num_groups: int = 32, - norm_eps: float = 1e-6, - spatial_padding_mode: str = "reflect", - ) -> None: - super().__init__() - self.resnets = nn.ModuleList( - [ - PH3ImageResnetBlock3d( - in_channels=in_channels if i == 0 else out_channels, - out_channels=out_channels, - norm_num_groups=norm_num_groups, - norm_eps=norm_eps, - spatial_padding_mode=spatial_padding_mode, - ) - for i in range(num_layers) - ] - ) - self.downsamplers = None - if temporal_downsample_factor * spatial_downsample_factor > 1: - self.downsamplers = nn.ModuleList( - [ - PH3ImageDownsample3d( - out_channels, - out_channels, - temporal_stride=temporal_downsample_factor, - spatial_stride=spatial_downsample_factor, - spatial_padding_mode=spatial_padding_mode, - ) - ] - ) - - self.gradient_checkpointing = False - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - for resnet in self.resnets: - if torch.is_grad_enabled() and self.gradient_checkpointing: - hidden_states = self._gradient_checkpointing_func(resnet, hidden_states) - else: - hidden_states = resnet(hidden_states) - if self.downsamplers is not None: - for downsampler in self.downsamplers: - hidden_states = downsampler(hidden_states) - return hidden_states - - -# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoEncoder3d with MiniMaxH3Video->PH3Image -class PH3ImageEncoder3d(nn.Module): - r""" - Causal 3D CNN encoder. `block_out_channels` gives the channel count of every level; the per-level - `spatial_downsample_factors` / `temporal_downsample_factors` multiply out to the total compression ratios. - """ - - def __init__( - self, - in_channels: int = 3, - out_channels: int = 48, - block_out_channels: tuple[int, ...] = (128, 256, 256, 512, 512, 1024), - layers_per_block: int = 2, - spatial_downsample_factors: tuple[int, ...] = (2, 2, 2, 2, 1, 1), - temporal_downsample_factors: tuple[int, ...] = (1, 2, 2, 1, 1, 1), - norm_num_groups: int = 32, - norm_eps: float = 1e-6, - spatial_padding_mode: str = "reflect", - ) -> None: - super().__init__() - - self.conv_in = PH3ImageCausalConv3d( - in_channels, - block_out_channels[0], - kernel_size=3, - spatial_padding=1, - temporal_padding=2, - spatial_padding_mode=spatial_padding_mode, - ) - - block_in_channels = (block_out_channels[0],) + tuple(block_out_channels[:-1]) - self.down_blocks = nn.ModuleList( - [ - PH3ImageDownBlock3d( - in_channels=block_in_channels[i], - out_channels=block_out_channels[i], - num_layers=layers_per_block, - temporal_downsample_factor=temporal_downsample_factors[i], - spatial_downsample_factor=spatial_downsample_factors[i], - norm_num_groups=norm_num_groups, - norm_eps=norm_eps, - spatial_padding_mode=spatial_padding_mode, - ) - for i in range(len(block_out_channels)) - ] - ) - - self.norm_out = PH3ImageGroupNorm(norm_num_groups, block_out_channels[-1], eps=norm_eps, affine=True) - self.conv_out = PH3ImageCausalConv3d( - block_out_channels[-1], - out_channels, - kernel_size=3, - spatial_padding=1, - temporal_padding=2, - spatial_padding_mode=spatial_padding_mode, - ) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - hidden_states = self.conv_in(hidden_states) - for down_block in self.down_blocks: - hidden_states = down_block(hidden_states) - hidden_states = F.silu(self.norm_out(hidden_states)) - return self.conv_out(hidden_states) - - -# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoRotaryPosEmbed with MiniMaxH3Video->PH3Image -class PH3ImageRotaryPosEmbed(nn.Module): - r""" - 3-axis rotary embedding for the ViT decoder. Coordinates are length-normalized to `[-1, 1)` per axis and scaled by - `2 * pi`, and the resulting `(t, h, w)` angles are concatenated and then duplicated, so the first `rope_dim_ratio * - attention_head_dim` channels of every head are rotated. - """ - - def __init__(self, dim: int, theta: float = 100.0, num_axes: int = 3) -> None: - super().__init__() - if dim % (2 * num_axes) != 0: - raise ValueError(f"`dim` {dim} must be divisible by `2 * num_axes` {2 * num_axes}.") - inv_freq = 1.0 / theta ** torch.arange(0, 1, 2 * num_axes / dim, dtype=torch.float32) - self.register_buffer("inv_freq", inv_freq, persistent=False) - - def forward(self, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - angles = 2.0 * math.pi * position_ids[:, :, :, None] * self.inv_freq[None, None, None, :] - angles = angles.flatten(2, 3).tile(2).unsqueeze(2) - return angles.cos(), angles.sin() - - -# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoAttnProcessor with MiniMaxH3Video->PH3Image -class PH3ImageAttnProcessor: - _attention_backend = None - _parallel_config = None - - def __call__( - self, - attn: "PH3ImageAttention", - hidden_states: torch.Tensor, - rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, - ) -> torch.Tensor: - query = attn.to_q(hidden_states).unflatten(2, (attn.heads, -1)) - key = attn.to_k(hidden_states).unflatten(2, (attn.heads, -1)) - value = attn.to_v(hidden_states).unflatten(2, (attn.heads, -1)) - - # The reference normalizes Q/K in float32 regardless of the compute dtype. - query = attn.norm_q(query.float()).to(query.dtype) - key = attn.norm_k(key.float()).to(key.dtype) - - if rotary_emb is not None: - cos, sin = rotary_emb - cos = cos.to(query.dtype) - sin = sin.to(query.dtype) - rotary_dim = cos.shape[-1] - query_rotary, query_pass = query[..., :rotary_dim], query[..., rotary_dim:] - key_rotary, key_pass = key[..., :rotary_dim], key[..., rotary_dim:] - query_first, query_second = query_rotary.chunk(2, dim=-1) - key_first, key_second = key_rotary.chunk(2, dim=-1) - query_rotated = torch.cat([-query_second, query_first], dim=-1) - key_rotated = torch.cat([-key_second, key_first], dim=-1) - query = torch.cat([query_rotary * cos + query_rotated * sin, query_pass], dim=-1) - key = torch.cat([key_rotary * cos + key_rotated * sin, key_pass], dim=-1) - - hidden_states = dispatch_attention_fn( - query, - key, - value, - attn_mask=None, - backend=self._attention_backend, - parallel_config=self._parallel_config, - ) - hidden_states = hidden_states.flatten(2, 3) - return attn.to_out[0](hidden_states) - - -# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoAttention with MiniMaxH3Video->PH3Image -class PH3ImageAttention(nn.Module, AttentionModuleMixin): - _default_processor_cls = PH3ImageAttnProcessor - _available_processors = [PH3ImageAttnProcessor] - - def __init__(self, dim: int, heads: int, dim_head: int, eps: float = 1e-5, bias: bool = True) -> None: - super().__init__() - self.heads = heads - self.dim_head = dim_head - self.use_bias = bias - inner_dim = heads * dim_head - - self.norm_q = nn.RMSNorm(dim_head, eps=eps, elementwise_affine=False) - self.norm_k = nn.RMSNorm(dim_head, eps=eps, elementwise_affine=False) - self.to_q = nn.Linear(dim, inner_dim, bias=bias) - self.to_k = nn.Linear(dim, inner_dim, bias=bias) - self.to_v = nn.Linear(dim, inner_dim, bias=bias) - self.to_out = nn.ModuleList([nn.Linear(inner_dim, dim, bias=bias), nn.Dropout(0.0)]) - - self.set_processor(PH3ImageAttnProcessor()) - - def forward( - self, hidden_states: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None - ) -> torch.Tensor: - return self.processor(self, hidden_states, rotary_emb) - - -# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoTransformerBlock with MiniMaxH3Video->PH3Image -class PH3ImageTransformerBlock(nn.Module): - def __init__( - self, - dim: int, - heads: int, - dim_head: int, - ffn_mult: int = 4, - eps: float = 1e-5, - bias: bool = True, - ) -> None: - super().__init__() - self.norm1 = nn.RMSNorm(dim, eps=eps, elementwise_affine=True) - self.attn = PH3ImageAttention(dim=dim, heads=heads, dim_head=dim_head, eps=eps, bias=bias) - self.scale1 = nn.Parameter(torch.zeros(dim)) - self.norm2 = nn.RMSNorm(dim, eps=eps, elementwise_affine=True) - self.ff = FeedForward(dim, mult=ffn_mult, activation_fn="swiglu", bias=bias) - self.scale2 = nn.Parameter(torch.zeros(dim)) - - def forward( - self, hidden_states: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None - ) -> torch.Tensor: - # The reference normalizes in float32 regardless of the compute dtype. The residual stream is float32 too - # because the anchors are, so the normed activation follows the projections instead. - compute_dtype = get_parameter_dtype(self.attn.to_q) - norm_hidden_states = self.norm1(hidden_states.float()).to(compute_dtype) - hidden_states = hidden_states + self.attn(norm_hidden_states, rotary_emb) * self.scale1 - norm_hidden_states = self.norm2(hidden_states.float()).to(compute_dtype) - hidden_states = hidden_states + self.ff(norm_hidden_states) * self.scale2 - return hidden_states - - -# Rotary time coordinate shared by every token of the decoded frame. -FRAME_POSITION = 2.0 * (0.5 / 7) - 1.0 - - -class PH3ImageViTDecoder(nn.Module): - r""" - A ViT decoder for a single latent frame. Each latent pixel becomes one token, decoded into a `patch_size x - patch_size` pixel block; all tokens share one time coordinate. - """ - - def __init__( - self, - in_channels: int = 24, - out_channels: int = 3, - patch_size: int = 16, - num_layers: int = 36, - num_attention_heads: int = 32, - attention_head_dim: int = 64, - num_register_tokens: int = 4, - ffn_mult: int = 4, - rope_theta: float = 100.0, - rope_dim_ratio: float = 0.75, - norm_eps: float = 1e-5, - ) -> None: - super().__init__() - dim = num_attention_heads * attention_head_dim - self.patch_size = patch_size - self.out_channels = out_channels - self.num_register_tokens = num_register_tokens - - self.rope = PH3ImageRotaryPosEmbed(int(attention_head_dim * rope_dim_ratio), theta=rope_theta) - self.proj_in = nn.Linear(in_channels, dim) - self.register_tokens = nn.Parameter(torch.zeros(1, num_register_tokens, dim)) - self.transformer_blocks = nn.ModuleList( - [ - PH3ImageTransformerBlock( - dim=dim, - heads=num_attention_heads, - dim_head=attention_head_dim, - ffn_mult=ffn_mult, - eps=norm_eps, - ) - for _ in range(num_layers) - ] - ) - self.norm_out = nn.LayerNorm(dim, elementwise_affine=True, eps=norm_eps) - self.proj_out = nn.Linear(dim, out_channels * patch_size * patch_size) - - self.gradient_checkpointing = False - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - batch_size, num_channels, _, height, width = hidden_states.shape - - hidden_states = hidden_states.permute(0, 2, 3, 4, 1).reshape(batch_size, height * width, num_channels) - hidden_states = self.proj_in(hidden_states) - num_patches = hidden_states.shape[1] - - register_tokens = self.register_tokens.expand(batch_size, -1, -1).to(hidden_states.dtype) - cls_token = torch.zeros_like(hidden_states[:, :1, :]) - hidden_states = torch.cat([hidden_states, register_tokens, cls_token], dim=1) - - grids = [torch.full((1,), FRAME_POSITION, device=hidden_states.device)] + [ - 2.0 * (torch.arange(0.5, size, dtype=torch.float32, device=hidden_states.device) / size) - 1.0 - for size in (height, width) - ] - position_ids = torch.stack(torch.meshgrid(*grids, indexing="ij"), dim=-1).flatten(0, 2) - position_ids = position_ids.unsqueeze(0).expand(batch_size, -1, -1) - suffix_ids = position_ids.new_zeros((batch_size, self.num_register_tokens + 1, 3)) - position_ids = torch.cat([position_ids, suffix_ids], dim=1) - rotary_emb = self.rope(position_ids) - - for block in self.transformer_blocks: - if torch.is_grad_enabled() and self.gradient_checkpointing: - hidden_states = self._gradient_checkpointing_func(block, hidden_states, rotary_emb) - else: - hidden_states = block(hidden_states, rotary_emb) - - hidden_states = self.norm_out(hidden_states).to(get_parameter_dtype(self.proj_out)) - hidden_states = self.proj_out(hidden_states) - hidden_states = hidden_states[:, :num_patches, :] - - patch_size = self.patch_size - hidden_states = hidden_states.view(batch_size, height, width, self.out_channels, patch_size, patch_size) - hidden_states = hidden_states.permute(0, 3, 1, 4, 2, 5).contiguous() - return hidden_states.reshape(batch_size, self.out_channels, 1, height * patch_size, width * patch_size) - - -class AutoencoderKLPH3Image(ModelMixin, ConfigMixin, AttentionMixin, AutoencoderMixin): - r""" - The image VAE of [P-H3-Image](https://huggingface.co/PrunaAI/P-H3-Image): MiniMax-H3's VAE encoder and latent - space, with a `PH3ImageViTDecoder` that decodes a single latent frame. - - Images are passed as single-frame videos, `(batch_size, channels, 1, height, width)`. As in - [`AutoencoderKLMiniMaxH3`], pixels are ImageNet-normalized RGB over `[0, 1]` (mean `(0.485, 0.456, 0.406)`, std - `(0.229, 0.224, 0.225)`) and spatial tiling is on by default. - """ - - _supports_gradient_checkpointing = True - _no_split_modules = ["PH3ImageResnetBlock3d", "PH3ImageTransformerBlock"] - _repeated_blocks = ["PH3ImageTransformerBlock"] - _skip_layerwise_casting_patterns = ["norm"] - # The encoder is float32, as in MiniMax-H3. The decoder's output projection is float32 so tiles blend at full - # precision. - _keep_in_fp32_modules = ["encoder", "quant_conv", "proj_out"] - - @register_to_config - def __init__( - self, - in_channels: int = 3, - out_channels: int = 3, - latent_channels: int = 24, - block_out_channels: tuple[int, ...] = (128, 256, 256, 512, 512, 1024), - layers_per_block: int = 2, - spatial_downsample_factors: tuple[int, ...] = (2, 2, 2, 2, 1, 1), - temporal_downsample_factors: tuple[int, ...] = (1, 2, 2, 1, 1, 1), - norm_num_groups: int = 32, - norm_eps: float = 1e-6, - spatial_padding_mode: str = "reflect", - decoder_num_layers: int = 36, - decoder_num_attention_heads: int = 32, - decoder_attention_head_dim: int = 64, - decoder_num_register_tokens: int = 4, - decoder_ffn_mult: int = 4, - decoder_rope_theta: float = 100.0, - decoder_rope_dim_ratio: float = 0.75, - decoder_norm_eps: float = 1e-5, - latents_mean: tuple[float, ...] = (0.0,) * 24, - latents_std: tuple[float, ...] = (1.0,) * 24, - ) -> None: - super().__init__() - - self.spatial_compression_ratio = math.prod(spatial_downsample_factors) - - self.encoder = PH3ImageEncoder3d( - in_channels=in_channels, - out_channels=2 * latent_channels, - block_out_channels=block_out_channels, - layers_per_block=layers_per_block, - spatial_downsample_factors=spatial_downsample_factors, - temporal_downsample_factors=temporal_downsample_factors, - norm_num_groups=norm_num_groups, - norm_eps=norm_eps, - spatial_padding_mode=spatial_padding_mode, - ) - self.quant_conv = nn.Conv3d(2 * latent_channels, 2 * latent_channels, kernel_size=1) - self.post_quant_conv = nn.Conv3d(latent_channels, latent_channels, kernel_size=1) - self.decoder = PH3ImageViTDecoder( - in_channels=latent_channels, - out_channels=out_channels, - patch_size=self.spatial_compression_ratio, - num_layers=decoder_num_layers, - num_attention_heads=decoder_num_attention_heads, - attention_head_dim=decoder_attention_head_dim, - num_register_tokens=decoder_num_register_tokens, - ffn_mult=decoder_ffn_mult, - rope_theta=decoder_rope_theta, - rope_dim_ratio=decoder_rope_dim_ratio, - norm_eps=decoder_norm_eps, - ) - - self.use_slicing = False - # Tile size and minimum overlap in pixels, as in `AutoencoderKLMiniMaxH3`. - self.use_tiling = True - self.tile_sample_min_height = 256 - self.tile_sample_min_width = 256 - self.tile_sample_min_overlap_height = 64 - self.tile_sample_min_overlap_width = 64 - - # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3.enable_tiling - def enable_tiling( - self, - tile_sample_min_height: int | None = None, - tile_sample_min_width: int | None = None, - tile_sample_min_overlap_height: int | None = None, - tile_sample_min_overlap_width: int | None = None, - ) -> None: - r""" - Enable tiled VAE encoding/decoding. When this option is enabled, the VAE splits the frames into tiles, encodes - or decodes each tile separately and linearly blends the overlaps back together. This lowers the memory - requirement and allows processing larger frames. - - Args: - tile_sample_min_height (`int`, *optional*): - The tile height in pixel space. Frames taller than this are split along the height dimension. - tile_sample_min_width (`int`, *optional*): - The tile width in pixel space. Frames wider than this are split along the width dimension. - tile_sample_min_overlap_height (`int`, *optional*): - The minimum overlap, in pixels, between two consecutive vertical tiles. - tile_sample_min_overlap_width (`int`, *optional*): - The minimum overlap, in pixels, between two consecutive horizontal tiles. - """ - self.use_tiling = True - self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height - self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width - self.tile_sample_min_overlap_height = tile_sample_min_overlap_height or self.tile_sample_min_overlap_height - self.tile_sample_min_overlap_width = tile_sample_min_overlap_width or self.tile_sample_min_overlap_width - - # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._split_tiles - def _split_tiles(self, length: int, tile_size: int, min_overlap: int) -> tuple[list[int], list[int], list[int]]: - r""" - Lay `tile_size`-wide tiles over `length` pixels. The number of tiles is the smallest one whose union can cover - `length` while keeping every overlap at least `min_overlap`; the slack is then distributed round-robin over the - overlaps in whole `spatial_compression_ratio` steps so that every tile boundary stays latent-aligned. - """ - if tile_size >= length: - return [0], [length], [] - - num_tiles = math.ceil(length / tile_size) - while tile_size * num_tiles - min_overlap * (num_tiles - 1) - length < 0: - num_tiles += 1 - - overlaps = [min_overlap] * (num_tiles - 1) - remaining = tile_size * num_tiles - sum(overlaps) - length - for i in range(remaining // self.spatial_compression_ratio): - overlaps[i % (num_tiles - 1)] += self.spatial_compression_ratio - - tile_start_indices = [0] - for i in range(num_tiles - 1): - tile_start_indices.append(tile_start_indices[-1] + tile_size - overlaps[i]) - return tile_start_indices, [tile_size] * num_tiles, overlaps - - # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._blend - def _blend(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int, dim: int) -> torch.Tensor: - blend_extent = min(a.shape[dim], b.shape[dim], blend_extent) - positions = torch.arange(blend_extent, device=b.device, dtype=b.dtype) - shape = [1] * a.ndim - shape[dim] = blend_extent - weight_a = (1 - positions / blend_extent).view(shape) - weight_b = (positions / blend_extent).view(shape) - - slice_a = [slice(None)] * a.ndim - slice_a[dim] = slice(-blend_extent, None) - slice_b = [slice(None)] * b.ndim - slice_b[dim] = slice(0, blend_extent) - blended = a[tuple(slice_a)] * weight_a + b[tuple(slice_b)] * weight_b - - if blend_extent == b.shape[dim]: - return blended - slice_rest = [slice(None)] * b.ndim - slice_rest[dim] = slice(blend_extent, None) - return torch.cat([blended, b[tuple(slice_rest)]], dim=dim) - - # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._stitch_tiles - def _stitch_tiles( - self, - tiles: list[list[torch.Tensor]], - height_overlaps: list[int], - width_overlaps: list[int], - ) -> torch.Tensor: - result_rows = [] - for i, row in enumerate(tiles): - result_row = [] - for j, tile in enumerate(row): - if i > 0: - tile = self._blend(tiles[i - 1][j], tile, height_overlaps[i - 1], dim=-2) - if j > 0: - tile = self._blend(row[j - 1], tile, width_overlaps[j - 1], dim=-1) - if i < len(tiles) - 1: - tile = tile[..., : -height_overlaps[i], :] - if j < len(row) - 1: - tile = tile[..., :, : -width_overlaps[j]] - result_row.append(tile) - result_rows.append(torch.cat(result_row, dim=-1)) - return torch.cat(result_rows, dim=-2) - - # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._encode_clip - def _encode_clip(self, x: torch.Tensor) -> torch.Tensor: - r"""Encode one temporal clip, spatially tiled when tiling is enabled.""" - if not self.use_tiling: - return self.quant_conv(self.encoder(x)) - - height, width = x.shape[-2], x.shape[-1] - y_indices, y_lengths, y_overlaps = self._split_tiles( - height, self.tile_sample_min_height, self.tile_sample_min_overlap_height - ) - x_indices, x_lengths, x_overlaps = self._split_tiles( - width, self.tile_sample_min_width, self.tile_sample_min_overlap_width - ) - - rows = [] - for i_pos, i_len in zip(y_indices, y_lengths): - row = [] - for j_pos, j_len in zip(x_indices, x_lengths): - tile = x[..., i_pos : i_pos + i_len, j_pos : j_pos + j_len] - row.append(self.quant_conv(self.encoder(tile))) - rows.append(row) - - latent_y_overlaps = [overlap // self.spatial_compression_ratio for overlap in y_overlaps] - latent_x_overlaps = [overlap // self.spatial_compression_ratio for overlap in x_overlaps] - return self._stitch_tiles(rows, latent_y_overlaps, latent_x_overlaps) - - # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._decode_clip - def _decode_clip(self, z: torch.Tensor) -> torch.Tensor: - r"""Decode one temporal clip, spatially tiled when tiling is enabled.""" - if not self.use_tiling: - return self.decoder(self.post_quant_conv(z)) - - # Tiles are laid out in pixel space and then mapped back onto the latent grid. - height = z.shape[-2] * self.spatial_compression_ratio - width = z.shape[-1] * self.spatial_compression_ratio - y_indices, y_lengths, y_overlaps = self._split_tiles( - height, self.tile_sample_min_height, self.tile_sample_min_overlap_height - ) - x_indices, x_lengths, x_overlaps = self._split_tiles( - width, self.tile_sample_min_width, self.tile_sample_min_overlap_width - ) - - ratio = self.spatial_compression_ratio - rows = [] - for i_pos, i_len in zip(y_indices, y_lengths): - row = [] - for j_pos, j_len in zip(x_indices, x_lengths): - tile = z[ - ..., - i_pos // ratio : i_pos // ratio + i_len // ratio, - j_pos // ratio : j_pos // ratio + j_len // ratio, - ] - row.append(self.decoder(self.post_quant_conv(tile))) - rows.append(row) - - return self._stitch_tiles(rows, y_overlaps, x_overlaps) - - @apply_forward_hook - def encode(self, x: torch.Tensor, return_dict: bool = True) -> AutoencoderKLOutput | tuple[torch.Tensor]: - r""" - Encode a batch of images into latents. - - Args: - x (`torch.Tensor`): - Input batch of images as single-frame videos, shape `(batch_size, in_channels, 1, height, width)`. - return_dict (`bool`, *optional*, defaults to `True`): - Whether to return a [`~models.autoencoders.autoencoder_kl.AutoencoderKLOutput`] instead of a plain - tuple. - - Returns: - The latent distribution of the encoded images. Note that the latents are normalized with `latents_mean` / - `latents_std` afterwards. - """ - if x.shape[2] != 1: - raise ValueError(f"`AutoencoderKLPH3Image` encodes single frames, got {x.shape[2]} frames.") - # The encoder is float32 whatever the model's dtype. - x = x.to(get_parameter_dtype(self.encoder)) - if self.use_slicing and x.shape[0] > 1: - moments = torch.cat([self._encode_clip(x_slice) for x_slice in x.split(1)]) - else: - moments = self._encode_clip(x) - posterior = DiagonalGaussianDistribution(moments) - if not return_dict: - return (posterior,) - return AutoencoderKLOutput(latent_dist=posterior) - - @apply_forward_hook - def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | tuple[torch.Tensor]: - r""" - Decode a batch of latent images. - - Args: - z (`torch.Tensor`): - Input batch of latents, shape `(batch_size, latent_channels, 1, height, width)`. - return_dict (`bool`, *optional*, defaults to `True`): - Whether to return a [`~models.autoencoders.vae.DecoderOutput`] instead of a plain tuple. - - Returns: - [`~models.autoencoders.vae.DecoderOutput`] or `tuple`: - The decoded images, shape `(batch_size, out_channels, 1, height, width)`. - """ - if z.shape[2] != 1: - raise ValueError(f"`AutoencoderKLPH3Image` decodes single latent frames, got {z.shape[2]} frames.") - # The decoder is mixed precision, so align with the first module that consumes the latents. - z = z.to(get_parameter_dtype(self.post_quant_conv)) - if self.use_slicing and z.shape[0] > 1: - decoded = torch.cat([self._decode_clip(z_slice) for z_slice in z.split(1)]) - else: - decoded = self._decode_clip(z) - if not return_dict: - return (decoded,) - return DecoderOutput(sample=decoded) - - def forward( - self, - sample: torch.Tensor, - sample_posterior: bool = False, - generator: torch.Generator | None = None, - return_dict: bool = True, - ) -> DecoderOutput | tuple[torch.Tensor]: - r""" - Encode then decode a batch of images. - - Args: - sample (`torch.Tensor`): - Input batch of images as single-frame videos, shape `(batch_size, in_channels, 1, height, width)`. - sample_posterior (`bool`, *optional*, defaults to `False`): - Whether to sample the posterior instead of taking its mode. - generator (`torch.Generator`, *optional*): - Generator used when `sample_posterior=True`. - return_dict (`bool`, *optional*, defaults to `True`): - Whether to return a [`~models.autoencoders.vae.DecoderOutput`] instead of a plain tuple. - - Returns: - [`~models.autoencoders.vae.DecoderOutput`] or `tuple`: - The round-tripped images, shape `(batch_size, out_channels, 1, height, width)`. - """ - posterior = self.encode(sample).latent_dist - z = posterior.sample(generator=generator) if sample_posterior else posterior.mode() - return self.decode(z, return_dict=return_dict) +from .autoencoder_kl_p_image_oss import AutoencoderKLPH3Image, AutoencoderKLPImageOSS # noqa: F401 diff --git a/src/diffusers/models/autoencoders/autoencoder_kl_p_image_oss.py b/src/diffusers/models/autoencoders/autoencoder_kl_p_image_oss.py new file mode 100644 index 000000000000..4aa1b1af5bfb --- /dev/null +++ b/src/diffusers/models/autoencoders/autoencoder_kl_p_image_oss.py @@ -0,0 +1,814 @@ +# Copyright 2026 Pruna AI and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ...configuration_utils import ConfigMixin, register_to_config +from ...utils.accelerate_utils import apply_forward_hook +from ..attention import AttentionMixin, AttentionModuleMixin, FeedForward +from ..attention_dispatch import dispatch_attention_fn +from ..modeling_outputs import AutoencoderKLOutput +from ..modeling_utils import ModelMixin, get_parameter_dtype +from .vae import AutoencoderMixin, DecoderOutput, DiagonalGaussianDistribution + + +# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoCausalConv3d with MiniMaxH3Video->PImageOSS +class PImageOSSCausalConv3d(nn.Conv3d): + r""" + 3D convolution used throughout the MiniMax-H3 video encoder. + + Spatial padding is symmetric and uses `spatial_padding_mode` (`"reflect"` in the released checkpoint); temporal + padding is causal, i.e. `kernel_size_t - 1` zero frames are prepended and nothing is appended. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int | tuple[int, int, int], + stride: int | tuple[int, int, int] = 1, + spatial_padding: int = 0, + temporal_padding: int = 0, + spatial_padding_mode: str = "reflect", + ) -> None: + super().__init__(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=0) + self.spatial_padding = spatial_padding + self.temporal_padding = temporal_padding + self.spatial_padding_mode = spatial_padding_mode + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if self.spatial_padding > 0: + padding = self.spatial_padding + hidden_states = F.pad( + hidden_states, (padding, padding, padding, padding, 0, 0), mode=self.spatial_padding_mode + ) + if self.temporal_padding > 0: + hidden_states = F.pad(hidden_states, (0, 0, 0, 0, self.temporal_padding, 0), mode="constant") + return F.conv3d(hidden_states, self.weight, self.bias, stride=self.stride, padding=0, dilation=self.dilation) + + +# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoGroupNorm with MiniMaxH3Video->PImageOSS +class PImageOSSGroupNorm(nn.GroupNorm): + r""" + Group normalization applied to each latent frame in isolation (`use_t_isolated_gn` in the original config): the + temporal axis is folded into the batch axis so statistics never mix across frames. + """ + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, num_frames, height, width = hidden_states.shape + hidden_states = hidden_states.permute(0, 2, 1, 3, 4).contiguous() + hidden_states = hidden_states.view(batch_size * num_frames, num_channels, 1, height, width) + hidden_states = super().forward(hidden_states) + hidden_states = hidden_states.view(batch_size, num_frames, num_channels, height, width) + return hidden_states.permute(0, 2, 1, 3, 4).contiguous() + + +# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoResnetBlock3d with MiniMaxH3Video->PImageOSS +class PImageOSSResnetBlock3d(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + norm_num_groups: int = 32, + norm_eps: float = 1e-6, + spatial_padding_mode: str = "reflect", + ) -> None: + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + self.norm1 = PImageOSSGroupNorm(norm_num_groups, in_channels, eps=norm_eps, affine=True) + self.conv1 = PImageOSSCausalConv3d( + in_channels, + out_channels, + kernel_size=3, + spatial_padding=1, + temporal_padding=2, + spatial_padding_mode=spatial_padding_mode, + ) + self.norm2 = PImageOSSGroupNorm(norm_num_groups, out_channels, eps=norm_eps, affine=True) + self.conv2 = PImageOSSCausalConv3d( + out_channels, + out_channels, + kernel_size=3, + spatial_padding=1, + temporal_padding=2, + spatial_padding_mode=spatial_padding_mode, + ) + self.conv_shortcut = None + if in_channels != out_channels: + self.conv_shortcut = PImageOSSCausalConv3d(in_channels, out_channels, kernel_size=1) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + residual = hidden_states + hidden_states = F.silu(self.norm1(hidden_states)) + hidden_states = self.conv1(hidden_states) + hidden_states = F.silu(self.norm2(hidden_states)) + hidden_states = self.conv2(hidden_states) + if self.conv_shortcut is not None: + residual = self.conv_shortcut(residual) + return residual + hidden_states + + +# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoDownsample3d with MiniMaxH3Video->PImageOSS +class PImageOSSDownsample3d(nn.Module): + r""" + Strided 3x3x3 downsampling convolution. A spatial stride of 2 is preceded by an asymmetric bottom/right pad of 1 + (the convolution itself carries no spatial padding), so the output is exactly `ceil(size / 2)`. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + temporal_stride: int = 1, + spatial_stride: int = 2, + spatial_padding_mode: str = "reflect", + ) -> None: + super().__init__() + self.spatial_stride = spatial_stride + self.spatial_padding_mode = spatial_padding_mode + self.conv = PImageOSSCausalConv3d( + in_channels, + out_channels, + kernel_size=3, + stride=(temporal_stride, spatial_stride, spatial_stride), + spatial_padding=0, + temporal_padding=2, + spatial_padding_mode=spatial_padding_mode, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if self.spatial_stride == 2: + hidden_states = F.pad(hidden_states, (0, 1, 0, 1, 0, 0), mode=self.spatial_padding_mode) + return self.conv(hidden_states) + + +# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoDownBlock3d with MiniMaxH3Video->PImageOSS +class PImageOSSDownBlock3d(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + num_layers: int, + temporal_downsample_factor: int, + spatial_downsample_factor: int, + norm_num_groups: int = 32, + norm_eps: float = 1e-6, + spatial_padding_mode: str = "reflect", + ) -> None: + super().__init__() + self.resnets = nn.ModuleList( + [ + PImageOSSResnetBlock3d( + in_channels=in_channels if i == 0 else out_channels, + out_channels=out_channels, + norm_num_groups=norm_num_groups, + norm_eps=norm_eps, + spatial_padding_mode=spatial_padding_mode, + ) + for i in range(num_layers) + ] + ) + self.downsamplers = None + if temporal_downsample_factor * spatial_downsample_factor > 1: + self.downsamplers = nn.ModuleList( + [ + PImageOSSDownsample3d( + out_channels, + out_channels, + temporal_stride=temporal_downsample_factor, + spatial_stride=spatial_downsample_factor, + spatial_padding_mode=spatial_padding_mode, + ) + ] + ) + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + for resnet in self.resnets: + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func(resnet, hidden_states) + else: + hidden_states = resnet(hidden_states) + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + return hidden_states + + +# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoEncoder3d with MiniMaxH3Video->PImageOSS +class PImageOSSEncoder3d(nn.Module): + r""" + Causal 3D CNN encoder. `block_out_channels` gives the channel count of every level; the per-level + `spatial_downsample_factors` / `temporal_downsample_factors` multiply out to the total compression ratios. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 48, + block_out_channels: tuple[int, ...] = (128, 256, 256, 512, 512, 1024), + layers_per_block: int = 2, + spatial_downsample_factors: tuple[int, ...] = (2, 2, 2, 2, 1, 1), + temporal_downsample_factors: tuple[int, ...] = (1, 2, 2, 1, 1, 1), + norm_num_groups: int = 32, + norm_eps: float = 1e-6, + spatial_padding_mode: str = "reflect", + ) -> None: + super().__init__() + + self.conv_in = PImageOSSCausalConv3d( + in_channels, + block_out_channels[0], + kernel_size=3, + spatial_padding=1, + temporal_padding=2, + spatial_padding_mode=spatial_padding_mode, + ) + + block_in_channels = (block_out_channels[0],) + tuple(block_out_channels[:-1]) + self.down_blocks = nn.ModuleList( + [ + PImageOSSDownBlock3d( + in_channels=block_in_channels[i], + out_channels=block_out_channels[i], + num_layers=layers_per_block, + temporal_downsample_factor=temporal_downsample_factors[i], + spatial_downsample_factor=spatial_downsample_factors[i], + norm_num_groups=norm_num_groups, + norm_eps=norm_eps, + spatial_padding_mode=spatial_padding_mode, + ) + for i in range(len(block_out_channels)) + ] + ) + + self.norm_out = PImageOSSGroupNorm(norm_num_groups, block_out_channels[-1], eps=norm_eps, affine=True) + self.conv_out = PImageOSSCausalConv3d( + block_out_channels[-1], + out_channels, + kernel_size=3, + spatial_padding=1, + temporal_padding=2, + spatial_padding_mode=spatial_padding_mode, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.conv_in(hidden_states) + for down_block in self.down_blocks: + hidden_states = down_block(hidden_states) + hidden_states = F.silu(self.norm_out(hidden_states)) + return self.conv_out(hidden_states) + + +# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoRotaryPosEmbed with MiniMaxH3Video->PImageOSS +class PImageOSSRotaryPosEmbed(nn.Module): + r""" + 3-axis rotary embedding for the ViT decoder. Coordinates are length-normalized to `[-1, 1)` per axis and scaled by + `2 * pi`, and the resulting `(t, h, w)` angles are concatenated and then duplicated, so the first `rope_dim_ratio * + attention_head_dim` channels of every head are rotated. + """ + + def __init__(self, dim: int, theta: float = 100.0, num_axes: int = 3) -> None: + super().__init__() + if dim % (2 * num_axes) != 0: + raise ValueError(f"`dim` {dim} must be divisible by `2 * num_axes` {2 * num_axes}.") + inv_freq = 1.0 / theta ** torch.arange(0, 1, 2 * num_axes / dim, dtype=torch.float32) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + angles = 2.0 * math.pi * position_ids[:, :, :, None] * self.inv_freq[None, None, None, :] + angles = angles.flatten(2, 3).tile(2).unsqueeze(2) + return angles.cos(), angles.sin() + + +# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoAttnProcessor with MiniMaxH3Video->PImageOSS +class PImageOSSAttnProcessor: + _attention_backend = None + _parallel_config = None + + def __call__( + self, + attn: "PImageOSSAttention", + hidden_states: torch.Tensor, + rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, + ) -> torch.Tensor: + query = attn.to_q(hidden_states).unflatten(2, (attn.heads, -1)) + key = attn.to_k(hidden_states).unflatten(2, (attn.heads, -1)) + value = attn.to_v(hidden_states).unflatten(2, (attn.heads, -1)) + + # The reference normalizes Q/K in float32 regardless of the compute dtype. + query = attn.norm_q(query.float()).to(query.dtype) + key = attn.norm_k(key.float()).to(key.dtype) + + if rotary_emb is not None: + cos, sin = rotary_emb + cos = cos.to(query.dtype) + sin = sin.to(query.dtype) + rotary_dim = cos.shape[-1] + query_rotary, query_pass = query[..., :rotary_dim], query[..., rotary_dim:] + key_rotary, key_pass = key[..., :rotary_dim], key[..., rotary_dim:] + query_first, query_second = query_rotary.chunk(2, dim=-1) + key_first, key_second = key_rotary.chunk(2, dim=-1) + query_rotated = torch.cat([-query_second, query_first], dim=-1) + key_rotated = torch.cat([-key_second, key_first], dim=-1) + query = torch.cat([query_rotary * cos + query_rotated * sin, query_pass], dim=-1) + key = torch.cat([key_rotary * cos + key_rotated * sin, key_pass], dim=-1) + + hidden_states = dispatch_attention_fn( + query, + key, + value, + attn_mask=None, + backend=self._attention_backend, + parallel_config=self._parallel_config, + ) + hidden_states = hidden_states.flatten(2, 3) + return attn.to_out[0](hidden_states) + + +# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoAttention with MiniMaxH3Video->PImageOSS +class PImageOSSAttention(nn.Module, AttentionModuleMixin): + _default_processor_cls = PImageOSSAttnProcessor + _available_processors = [PImageOSSAttnProcessor] + + def __init__(self, dim: int, heads: int, dim_head: int, eps: float = 1e-5, bias: bool = True) -> None: + super().__init__() + self.heads = heads + self.dim_head = dim_head + self.use_bias = bias + inner_dim = heads * dim_head + + self.norm_q = nn.RMSNorm(dim_head, eps=eps, elementwise_affine=False) + self.norm_k = nn.RMSNorm(dim_head, eps=eps, elementwise_affine=False) + self.to_q = nn.Linear(dim, inner_dim, bias=bias) + self.to_k = nn.Linear(dim, inner_dim, bias=bias) + self.to_v = nn.Linear(dim, inner_dim, bias=bias) + self.to_out = nn.ModuleList([nn.Linear(inner_dim, dim, bias=bias), nn.Dropout(0.0)]) + + self.set_processor(PImageOSSAttnProcessor()) + + def forward( + self, hidden_states: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None + ) -> torch.Tensor: + return self.processor(self, hidden_states, rotary_emb) + + +# Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.MiniMaxH3VideoTransformerBlock with MiniMaxH3Video->PImageOSS +class PImageOSSTransformerBlock(nn.Module): + def __init__( + self, + dim: int, + heads: int, + dim_head: int, + ffn_mult: int = 4, + eps: float = 1e-5, + bias: bool = True, + ) -> None: + super().__init__() + self.norm1 = nn.RMSNorm(dim, eps=eps, elementwise_affine=True) + self.attn = PImageOSSAttention(dim=dim, heads=heads, dim_head=dim_head, eps=eps, bias=bias) + self.scale1 = nn.Parameter(torch.zeros(dim)) + self.norm2 = nn.RMSNorm(dim, eps=eps, elementwise_affine=True) + self.ff = FeedForward(dim, mult=ffn_mult, activation_fn="swiglu", bias=bias) + self.scale2 = nn.Parameter(torch.zeros(dim)) + + def forward( + self, hidden_states: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None + ) -> torch.Tensor: + # The reference normalizes in float32 regardless of the compute dtype. The residual stream is float32 too + # because the anchors are, so the normed activation follows the projections instead. + compute_dtype = get_parameter_dtype(self.attn.to_q) + norm_hidden_states = self.norm1(hidden_states.float()).to(compute_dtype) + hidden_states = hidden_states + self.attn(norm_hidden_states, rotary_emb) * self.scale1 + norm_hidden_states = self.norm2(hidden_states.float()).to(compute_dtype) + hidden_states = hidden_states + self.ff(norm_hidden_states) * self.scale2 + return hidden_states + + +# Rotary time coordinate shared by every token of the decoded frame. +FRAME_POSITION = 2.0 * (0.5 / 7) - 1.0 + + +class PImageOSSViTDecoder(nn.Module): + r""" + A ViT decoder for a single latent frame. Each latent pixel becomes one token, decoded into a `patch_size x + patch_size` pixel block; all tokens share one time coordinate. + """ + + def __init__( + self, + in_channels: int = 24, + out_channels: int = 3, + patch_size: int = 16, + num_layers: int = 36, + num_attention_heads: int = 32, + attention_head_dim: int = 64, + num_register_tokens: int = 4, + ffn_mult: int = 4, + rope_theta: float = 100.0, + rope_dim_ratio: float = 0.75, + norm_eps: float = 1e-5, + ) -> None: + super().__init__() + dim = num_attention_heads * attention_head_dim + self.patch_size = patch_size + self.out_channels = out_channels + self.num_register_tokens = num_register_tokens + + self.rope = PImageOSSRotaryPosEmbed(int(attention_head_dim * rope_dim_ratio), theta=rope_theta) + self.proj_in = nn.Linear(in_channels, dim) + self.register_tokens = nn.Parameter(torch.zeros(1, num_register_tokens, dim)) + self.transformer_blocks = nn.ModuleList( + [ + PImageOSSTransformerBlock( + dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + ffn_mult=ffn_mult, + eps=norm_eps, + ) + for _ in range(num_layers) + ] + ) + self.norm_out = nn.LayerNorm(dim, elementwise_affine=True, eps=norm_eps) + self.proj_out = nn.Linear(dim, out_channels * patch_size * patch_size) + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, _, height, width = hidden_states.shape + + hidden_states = hidden_states.permute(0, 2, 3, 4, 1).reshape(batch_size, height * width, num_channels) + hidden_states = self.proj_in(hidden_states) + num_patches = hidden_states.shape[1] + + register_tokens = self.register_tokens.expand(batch_size, -1, -1).to(hidden_states.dtype) + cls_token = torch.zeros_like(hidden_states[:, :1, :]) + hidden_states = torch.cat([hidden_states, register_tokens, cls_token], dim=1) + + grids = [torch.full((1,), FRAME_POSITION, device=hidden_states.device)] + [ + 2.0 * (torch.arange(0.5, size, dtype=torch.float32, device=hidden_states.device) / size) - 1.0 + for size in (height, width) + ] + position_ids = torch.stack(torch.meshgrid(*grids, indexing="ij"), dim=-1).flatten(0, 2) + position_ids = position_ids.unsqueeze(0).expand(batch_size, -1, -1) + suffix_ids = position_ids.new_zeros((batch_size, self.num_register_tokens + 1, 3)) + position_ids = torch.cat([position_ids, suffix_ids], dim=1) + rotary_emb = self.rope(position_ids) + + for block in self.transformer_blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func(block, hidden_states, rotary_emb) + else: + hidden_states = block(hidden_states, rotary_emb) + + hidden_states = self.norm_out(hidden_states).to(get_parameter_dtype(self.proj_out)) + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states[:, :num_patches, :] + + patch_size = self.patch_size + hidden_states = hidden_states.view(batch_size, height, width, self.out_channels, patch_size, patch_size) + hidden_states = hidden_states.permute(0, 3, 1, 4, 2, 5).contiguous() + return hidden_states.reshape(batch_size, self.out_channels, 1, height * patch_size, width * patch_size) + + +class AutoencoderKLPImageOSS(ModelMixin, ConfigMixin, AttentionMixin, AutoencoderMixin): + r""" + The image VAE of [P-Image-OSS](https://huggingface.co/PrunaAI/P-Image-OSS): MiniMax-H3's VAE encoder and latent + space, with a `PImageOSSViTDecoder` that decodes a single latent frame. + + Images are passed as single-frame videos, `(batch_size, channels, 1, height, width)`. As in + [`AutoencoderKLMiniMaxH3`], pixels are ImageNet-normalized RGB over `[0, 1]` (mean `(0.485, 0.456, 0.406)`, std + `(0.229, 0.224, 0.225)`) and spatial tiling is on by default. + """ + + _supports_gradient_checkpointing = True + _no_split_modules = ["PImageOSSResnetBlock3d", "PImageOSSTransformerBlock"] + _repeated_blocks = ["PImageOSSTransformerBlock"] + _skip_layerwise_casting_patterns = ["norm"] + # The encoder is float32, as in MiniMax-H3. The decoder's output projection is float32 so tiles blend at full + # precision. + _keep_in_fp32_modules = ["encoder", "quant_conv", "proj_out"] + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + latent_channels: int = 24, + block_out_channels: tuple[int, ...] = (128, 256, 256, 512, 512, 1024), + layers_per_block: int = 2, + spatial_downsample_factors: tuple[int, ...] = (2, 2, 2, 2, 1, 1), + temporal_downsample_factors: tuple[int, ...] = (1, 2, 2, 1, 1, 1), + norm_num_groups: int = 32, + norm_eps: float = 1e-6, + spatial_padding_mode: str = "reflect", + decoder_num_layers: int = 36, + decoder_num_attention_heads: int = 32, + decoder_attention_head_dim: int = 64, + decoder_num_register_tokens: int = 4, + decoder_ffn_mult: int = 4, + decoder_rope_theta: float = 100.0, + decoder_rope_dim_ratio: float = 0.75, + decoder_norm_eps: float = 1e-5, + latents_mean: tuple[float, ...] = (0.0,) * 24, + latents_std: tuple[float, ...] = (1.0,) * 24, + ) -> None: + super().__init__() + + self.spatial_compression_ratio = math.prod(spatial_downsample_factors) + + self.encoder = PImageOSSEncoder3d( + in_channels=in_channels, + out_channels=2 * latent_channels, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + spatial_downsample_factors=spatial_downsample_factors, + temporal_downsample_factors=temporal_downsample_factors, + norm_num_groups=norm_num_groups, + norm_eps=norm_eps, + spatial_padding_mode=spatial_padding_mode, + ) + self.quant_conv = nn.Conv3d(2 * latent_channels, 2 * latent_channels, kernel_size=1) + self.post_quant_conv = nn.Conv3d(latent_channels, latent_channels, kernel_size=1) + self.decoder = PImageOSSViTDecoder( + in_channels=latent_channels, + out_channels=out_channels, + patch_size=self.spatial_compression_ratio, + num_layers=decoder_num_layers, + num_attention_heads=decoder_num_attention_heads, + attention_head_dim=decoder_attention_head_dim, + num_register_tokens=decoder_num_register_tokens, + ffn_mult=decoder_ffn_mult, + rope_theta=decoder_rope_theta, + rope_dim_ratio=decoder_rope_dim_ratio, + norm_eps=decoder_norm_eps, + ) + + self.use_slicing = False + # Tile size and minimum overlap in pixels, as in `AutoencoderKLMiniMaxH3`. + self.use_tiling = True + self.tile_sample_min_height = 256 + self.tile_sample_min_width = 256 + self.tile_sample_min_overlap_height = 64 + self.tile_sample_min_overlap_width = 64 + + # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3.enable_tiling + def enable_tiling( + self, + tile_sample_min_height: int | None = None, + tile_sample_min_width: int | None = None, + tile_sample_min_overlap_height: int | None = None, + tile_sample_min_overlap_width: int | None = None, + ) -> None: + r""" + Enable tiled VAE encoding/decoding. When this option is enabled, the VAE splits the frames into tiles, encodes + or decodes each tile separately and linearly blends the overlaps back together. This lowers the memory + requirement and allows processing larger frames. + + Args: + tile_sample_min_height (`int`, *optional*): + The tile height in pixel space. Frames taller than this are split along the height dimension. + tile_sample_min_width (`int`, *optional*): + The tile width in pixel space. Frames wider than this are split along the width dimension. + tile_sample_min_overlap_height (`int`, *optional*): + The minimum overlap, in pixels, between two consecutive vertical tiles. + tile_sample_min_overlap_width (`int`, *optional*): + The minimum overlap, in pixels, between two consecutive horizontal tiles. + """ + self.use_tiling = True + self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height + self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width + self.tile_sample_min_overlap_height = tile_sample_min_overlap_height or self.tile_sample_min_overlap_height + self.tile_sample_min_overlap_width = tile_sample_min_overlap_width or self.tile_sample_min_overlap_width + + # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._split_tiles + def _split_tiles(self, length: int, tile_size: int, min_overlap: int) -> tuple[list[int], list[int], list[int]]: + r""" + Lay `tile_size`-wide tiles over `length` pixels. The number of tiles is the smallest one whose union can cover + `length` while keeping every overlap at least `min_overlap`; the slack is then distributed round-robin over the + overlaps in whole `spatial_compression_ratio` steps so that every tile boundary stays latent-aligned. + """ + if tile_size >= length: + return [0], [length], [] + + num_tiles = math.ceil(length / tile_size) + while tile_size * num_tiles - min_overlap * (num_tiles - 1) - length < 0: + num_tiles += 1 + + overlaps = [min_overlap] * (num_tiles - 1) + remaining = tile_size * num_tiles - sum(overlaps) - length + for i in range(remaining // self.spatial_compression_ratio): + overlaps[i % (num_tiles - 1)] += self.spatial_compression_ratio + + tile_start_indices = [0] + for i in range(num_tiles - 1): + tile_start_indices.append(tile_start_indices[-1] + tile_size - overlaps[i]) + return tile_start_indices, [tile_size] * num_tiles, overlaps + + # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._blend + def _blend(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int, dim: int) -> torch.Tensor: + blend_extent = min(a.shape[dim], b.shape[dim], blend_extent) + positions = torch.arange(blend_extent, device=b.device, dtype=b.dtype) + shape = [1] * a.ndim + shape[dim] = blend_extent + weight_a = (1 - positions / blend_extent).view(shape) + weight_b = (positions / blend_extent).view(shape) + + slice_a = [slice(None)] * a.ndim + slice_a[dim] = slice(-blend_extent, None) + slice_b = [slice(None)] * b.ndim + slice_b[dim] = slice(0, blend_extent) + blended = a[tuple(slice_a)] * weight_a + b[tuple(slice_b)] * weight_b + + if blend_extent == b.shape[dim]: + return blended + slice_rest = [slice(None)] * b.ndim + slice_rest[dim] = slice(blend_extent, None) + return torch.cat([blended, b[tuple(slice_rest)]], dim=dim) + + # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._stitch_tiles + def _stitch_tiles( + self, + tiles: list[list[torch.Tensor]], + height_overlaps: list[int], + width_overlaps: list[int], + ) -> torch.Tensor: + result_rows = [] + for i, row in enumerate(tiles): + result_row = [] + for j, tile in enumerate(row): + if i > 0: + tile = self._blend(tiles[i - 1][j], tile, height_overlaps[i - 1], dim=-2) + if j > 0: + tile = self._blend(row[j - 1], tile, width_overlaps[j - 1], dim=-1) + if i < len(tiles) - 1: + tile = tile[..., : -height_overlaps[i], :] + if j < len(row) - 1: + tile = tile[..., :, : -width_overlaps[j]] + result_row.append(tile) + result_rows.append(torch.cat(result_row, dim=-1)) + return torch.cat(result_rows, dim=-2) + + # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._encode_clip + def _encode_clip(self, x: torch.Tensor) -> torch.Tensor: + r"""Encode one temporal clip, spatially tiled when tiling is enabled.""" + if not self.use_tiling: + return self.quant_conv(self.encoder(x)) + + height, width = x.shape[-2], x.shape[-1] + y_indices, y_lengths, y_overlaps = self._split_tiles( + height, self.tile_sample_min_height, self.tile_sample_min_overlap_height + ) + x_indices, x_lengths, x_overlaps = self._split_tiles( + width, self.tile_sample_min_width, self.tile_sample_min_overlap_width + ) + + rows = [] + for i_pos, i_len in zip(y_indices, y_lengths): + row = [] + for j_pos, j_len in zip(x_indices, x_lengths): + tile = x[..., i_pos : i_pos + i_len, j_pos : j_pos + j_len] + row.append(self.quant_conv(self.encoder(tile))) + rows.append(row) + + latent_y_overlaps = [overlap // self.spatial_compression_ratio for overlap in y_overlaps] + latent_x_overlaps = [overlap // self.spatial_compression_ratio for overlap in x_overlaps] + return self._stitch_tiles(rows, latent_y_overlaps, latent_x_overlaps) + + # Copied from diffusers.models.autoencoders.autoencoder_kl_minimax_h3.AutoencoderKLMiniMaxH3._decode_clip + def _decode_clip(self, z: torch.Tensor) -> torch.Tensor: + r"""Decode one temporal clip, spatially tiled when tiling is enabled.""" + if not self.use_tiling: + return self.decoder(self.post_quant_conv(z)) + + # Tiles are laid out in pixel space and then mapped back onto the latent grid. + height = z.shape[-2] * self.spatial_compression_ratio + width = z.shape[-1] * self.spatial_compression_ratio + y_indices, y_lengths, y_overlaps = self._split_tiles( + height, self.tile_sample_min_height, self.tile_sample_min_overlap_height + ) + x_indices, x_lengths, x_overlaps = self._split_tiles( + width, self.tile_sample_min_width, self.tile_sample_min_overlap_width + ) + + ratio = self.spatial_compression_ratio + rows = [] + for i_pos, i_len in zip(y_indices, y_lengths): + row = [] + for j_pos, j_len in zip(x_indices, x_lengths): + tile = z[ + ..., + i_pos // ratio : i_pos // ratio + i_len // ratio, + j_pos // ratio : j_pos // ratio + j_len // ratio, + ] + row.append(self.decoder(self.post_quant_conv(tile))) + rows.append(row) + + return self._stitch_tiles(rows, y_overlaps, x_overlaps) + + @apply_forward_hook + def encode(self, x: torch.Tensor, return_dict: bool = True) -> AutoencoderKLOutput | tuple[torch.Tensor]: + r""" + Encode a batch of images into latents. + + Args: + x (`torch.Tensor`): + Input batch of images as single-frame videos, shape `(batch_size, in_channels, 1, height, width)`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoders.autoencoder_kl.AutoencoderKLOutput`] instead of a plain + tuple. + + Returns: + The latent distribution of the encoded images. Note that the latents are normalized with `latents_mean` / + `latents_std` afterwards. + """ + if x.shape[2] != 1: + raise ValueError(f"`AutoencoderKLPImageOSS` encodes single frames, got {x.shape[2]} frames.") + # The encoder is float32 whatever the model's dtype. + x = x.to(get_parameter_dtype(self.encoder)) + if self.use_slicing and x.shape[0] > 1: + moments = torch.cat([self._encode_clip(x_slice) for x_slice in x.split(1)]) + else: + moments = self._encode_clip(x) + posterior = DiagonalGaussianDistribution(moments) + if not return_dict: + return (posterior,) + return AutoencoderKLOutput(latent_dist=posterior) + + @apply_forward_hook + def decode(self, z: torch.Tensor, return_dict: bool = True) -> DecoderOutput | tuple[torch.Tensor]: + r""" + Decode a batch of latent images. + + Args: + z (`torch.Tensor`): + Input batch of latents, shape `(batch_size, latent_channels, 1, height, width)`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoders.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.autoencoders.vae.DecoderOutput`] or `tuple`: + The decoded images, shape `(batch_size, out_channels, 1, height, width)`. + """ + if z.shape[2] != 1: + raise ValueError(f"`AutoencoderKLPImageOSS` decodes single latent frames, got {z.shape[2]} frames.") + # The decoder is mixed precision, so align with the first module that consumes the latents. + z = z.to(get_parameter_dtype(self.post_quant_conv)) + if self.use_slicing and z.shape[0] > 1: + decoded = torch.cat([self._decode_clip(z_slice) for z_slice in z.split(1)]) + else: + decoded = self._decode_clip(z) + if not return_dict: + return (decoded,) + return DecoderOutput(sample=decoded) + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + generator: torch.Generator | None = None, + return_dict: bool = True, + ) -> DecoderOutput | tuple[torch.Tensor]: + r""" + Encode then decode a batch of images. + + Args: + sample (`torch.Tensor`): + Input batch of images as single-frame videos, shape `(batch_size, in_channels, 1, height, width)`. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample the posterior instead of taking its mode. + generator (`torch.Generator`, *optional*): + Generator used when `sample_posterior=True`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoders.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.autoencoders.vae.DecoderOutput`] or `tuple`: + The round-tripped images, shape `(batch_size, out_channels, 1, height, width)`. + """ + posterior = self.encode(sample).latent_dist + z = posterior.sample(generator=generator) if sample_posterior else posterior.mode() + return self.decode(z, return_dict=return_dict) + + +# Deprecated alias for the former P-H3-Image name, kept so existing code and Hub configs keep loading. +AutoencoderKLPH3Image = AutoencoderKLPImageOSS diff --git a/src/diffusers/models/condition_embedders/__init__.py b/src/diffusers/models/condition_embedders/__init__.py index 229c2a7ee564..482f2cc5a6b3 100644 --- a/src/diffusers/models/condition_embedders/__init__.py +++ b/src/diffusers/models/condition_embedders/__init__.py @@ -9,7 +9,7 @@ from .condition_embedder_gligen import CLIPImageProjection from .condition_embedder_ltx2 import LTX2TextConnectors from .condition_embedder_minimax_music3 import MiniMaxMusic3ConditionEncoder - from .condition_embedder_p_h3_image import PH3ImageTextConnector + from .condition_embedder_p_image_oss import PH3ImageTextConnector, PImageOSSTextConnector from .condition_embedder_stable_audio import StableAudioProjectionModel from .condition_embedder_stable_audio_3 import StableAudio3DurationEmbedder from .condition_embedder_stable_unclip import StableUnCLIPImageNormalizer diff --git a/src/diffusers/models/condition_embedders/condition_embedder_p_h3_image.py b/src/diffusers/models/condition_embedders/condition_embedder_p_h3_image.py index 06c299c59972..234e634b1154 100644 --- a/src/diffusers/models/condition_embedders/condition_embedder_p_h3_image.py +++ b/src/diffusers/models/condition_embedders/condition_embedder_p_h3_image.py @@ -11,67 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import torch -import torch.nn as nn - -from ...configuration_utils import ConfigMixin, register_to_config -from ..modeling_utils import ModelMixin - - -class PH3ImageTextConnector(ModelMixin, ConfigMixin): - r""" - Maps hidden states of P-H3-Image's Qwen3-VL text encoder to the text conditioning its transformer reads. - - The hidden states listed in `hidden_state_indices` are concatenated, standardized, projected linearly and - destandardized. The first token is an attention sink with the same conditioning for every prompt, so it is replaced - by `sink_embedding`. - - Args: - hidden_state_indices (`tuple[int, ...]`, defaults to `(13, 19, 25, 31, 36)`): - Which entries of the text encoder's `hidden_states` are read, in the numbering of `output_hidden_states` - (`0` is the embedding output, `i` the output of decoder layer `i`, and the last one the final normed hidden - state). - text_hidden_size (`int`, defaults to `2560`): - The hidden size of the text encoder. - out_channels (`int`, defaults to `5120`): - The number of channels of the conditioning MiniMax-H3's transformer reads. - """ - - # Float32, as in the released checkpoint. - _keep_in_fp32_modules = ["input_mean", "input_std", "proj", "output_mean", "output_std", "sink_embedding"] - - @register_to_config - def __init__( - self, - hidden_state_indices: tuple[int, ...] = (13, 19, 25, 31, 36), - text_hidden_size: int = 2560, - out_channels: int = 5120, - ) -> None: - super().__init__() - in_channels = len(hidden_state_indices) * text_hidden_size - - self.input_mean = nn.Parameter(torch.zeros(in_channels)) - self.input_std = nn.Parameter(torch.ones(in_channels)) - self.proj = nn.Linear(in_channels, out_channels, bias=False) - self.output_mean = nn.Parameter(torch.zeros(out_channels)) - self.output_std = nn.Parameter(torch.ones(out_channels)) - self.sink_embedding = nn.Parameter(torch.zeros(out_channels)) - - def forward(self, hidden_states: tuple[torch.Tensor, ...]) -> torch.Tensor: - r""" - Args: - hidden_states (`tuple[torch.Tensor, ...]`): - The hidden states of the text encoder, as `output_hidden_states=True` returns them: one tensor of shape - `(batch_size, num_tokens, text_hidden_size)` per entry. - - Returns: - `torch.Tensor`: the conditioning, of shape `(batch_size, num_tokens, out_channels)`, in the dtype of the - connector's parameters. - """ - dtype = self.dtype - hidden_states = torch.cat([hidden_states[index] for index in self.config.hidden_state_indices], dim=-1) - hidden_states = (hidden_states.to(dtype) - self.input_mean) / self.input_std - hidden_states = self.proj(hidden_states) * self.output_std + self.output_mean - sink = self.sink_embedding.expand(hidden_states.shape[0], 1, -1) - return torch.cat([sink, hidden_states[:, 1:]], dim=1) +from .condition_embedder_p_image_oss import PH3ImageTextConnector, PImageOSSTextConnector # noqa: F401 diff --git a/src/diffusers/models/condition_embedders/condition_embedder_p_image_oss.py b/src/diffusers/models/condition_embedders/condition_embedder_p_image_oss.py new file mode 100644 index 000000000000..f79f232f5f62 --- /dev/null +++ b/src/diffusers/models/condition_embedders/condition_embedder_p_image_oss.py @@ -0,0 +1,81 @@ +# Copyright 2026 Pruna AI and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ..modeling_utils import ModelMixin + + +class PImageOSSTextConnector(ModelMixin, ConfigMixin): + r""" + Maps hidden states of P-Image-OSS's Qwen3-VL text encoder to the text conditioning its transformer reads. + + The hidden states listed in `hidden_state_indices` are concatenated, standardized, projected linearly and + destandardized. The first token is an attention sink with the same conditioning for every prompt, so it is replaced + by `sink_embedding`. + + Args: + hidden_state_indices (`tuple[int, ...]`, defaults to `(13, 19, 25, 31, 36)`): + Which entries of the text encoder's `hidden_states` are read, in the numbering of `output_hidden_states` + (`0` is the embedding output, `i` the output of decoder layer `i`, and the last one the final normed hidden + state). + text_hidden_size (`int`, defaults to `2560`): + The hidden size of the text encoder. + out_channels (`int`, defaults to `5120`): + The number of channels of the conditioning MiniMax-H3's transformer reads. + """ + + # Float32, as in the released checkpoint. + _keep_in_fp32_modules = ["input_mean", "input_std", "proj", "output_mean", "output_std", "sink_embedding"] + + @register_to_config + def __init__( + self, + hidden_state_indices: tuple[int, ...] = (13, 19, 25, 31, 36), + text_hidden_size: int = 2560, + out_channels: int = 5120, + ) -> None: + super().__init__() + in_channels = len(hidden_state_indices) * text_hidden_size + + self.input_mean = nn.Parameter(torch.zeros(in_channels)) + self.input_std = nn.Parameter(torch.ones(in_channels)) + self.proj = nn.Linear(in_channels, out_channels, bias=False) + self.output_mean = nn.Parameter(torch.zeros(out_channels)) + self.output_std = nn.Parameter(torch.ones(out_channels)) + self.sink_embedding = nn.Parameter(torch.zeros(out_channels)) + + def forward(self, hidden_states: tuple[torch.Tensor, ...]) -> torch.Tensor: + r""" + Args: + hidden_states (`tuple[torch.Tensor, ...]`): + The hidden states of the text encoder, as `output_hidden_states=True` returns them: one tensor of shape + `(batch_size, num_tokens, text_hidden_size)` per entry. + + Returns: + `torch.Tensor`: the conditioning, of shape `(batch_size, num_tokens, out_channels)`, in the dtype of the + connector's parameters. + """ + dtype = self.dtype + hidden_states = torch.cat([hidden_states[index] for index in self.config.hidden_state_indices], dim=-1) + hidden_states = (hidden_states.to(dtype) - self.input_mean) / self.input_std + hidden_states = self.proj(hidden_states) * self.output_std + self.output_mean + sink = self.sink_embedding.expand(hidden_states.shape[0], 1, -1) + return torch.cat([sink, hidden_states[:, 1:]], dim=1) + + +# Deprecated alias for the former P-H3-Image name, kept so existing code and Hub configs keep loading. +PH3ImageTextConnector = PImageOSSTextConnector diff --git a/src/diffusers/modular_pipelines/__init__.py b/src/diffusers/modular_pipelines/__init__.py index 11b4e92bac1d..fcaf8327870a 100644 --- a/src/diffusers/modular_pipelines/__init__.py +++ b/src/diffusers/modular_pipelines/__init__.py @@ -168,9 +168,11 @@ "PAN2ModularPipeline", "PAN2SafetyChecker", ] - _import_structure["p_h3_image"] = [ + _import_structure["p_image_oss"] = [ "PH3ImageBlocks", "PH3ImageModularPipeline", + "PImageOSSBlocks", + "PImageOSSModularPipeline", ] _import_structure["pika_h3_flex"] = [ "PikaH3FlexBlocks", @@ -267,9 +269,11 @@ SequentialPipelineBlocks, ) from .modular_pipeline_utils import ComponentSpec, ConfigSpec, InputParam, InsertableDict, OutputParam - from .p_h3_image import ( + from .p_image_oss import ( PH3ImageBlocks, PH3ImageModularPipeline, + PImageOSSBlocks, + PImageOSSModularPipeline, ) from .pan2 import PAN2Blocks, PAN2ModularPipeline, PAN2SafetyChecker from .pika_h3_flex import ( diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 1d1412362bbc..d7c8eb4a9ae0 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -221,7 +221,9 @@ def _helios_pyramid_map_fn(config_dict=None): ("minimax-music3", _create_default_map_fn("MiniMaxMusic3ModularPipeline")), ("pan2", _create_default_map_fn("PAN2ModularPipeline")), ("pika-h3-flex", _create_default_map_fn("PikaH3FlexModularPipeline")), - ("p-h3-image", _create_default_map_fn("PH3ImageModularPipeline")), + ("p-image-oss", _create_default_map_fn("PImageOSSModularPipeline")), + # Deprecated: former P-H3-Image model name. + ("p-h3-image", _create_default_map_fn("PImageOSSModularPipeline")), ("ernie-image", _create_default_map_fn("ErnieImageModularPipeline")), ] ) diff --git a/src/diffusers/modular_pipelines/p_h3_image/__init__.py b/src/diffusers/modular_pipelines/p_h3_image/__init__.py index 8b60ee0c74c6..7e497af82fe6 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/__init__.py +++ b/src/diffusers/modular_pipelines/p_h3_image/__init__.py @@ -1,47 +1,14 @@ -from typing import TYPE_CHECKING - -from ...utils import ( - DIFFUSERS_SLOW_IMPORT, - OptionalDependencyNotAvailable, - _LazyModule, - get_objects_from_module, - is_torch_available, - is_transformers_available, -) - - -_dummy_objects = {} -_import_structure = {} - -try: - if not (is_transformers_available() and is_torch_available()): - raise OptionalDependencyNotAvailable() -except OptionalDependencyNotAvailable: - from ...utils import dummy_torch_and_transformers_objects # noqa F403 - - _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) -else: - _import_structure["modular_blocks_p_h3_image"] = ["PH3ImageBlocks"] - _import_structure["modular_pipeline"] = ["PH3ImageModularPipeline"] - -if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: - try: - if not (is_transformers_available() and is_torch_available()): - raise OptionalDependencyNotAvailable() - except OptionalDependencyNotAvailable: - from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 - else: - from .modular_blocks_p_h3_image import PH3ImageBlocks - from .modular_pipeline import PH3ImageModularPipeline -else: - import sys - - sys.modules[__name__] = _LazyModule( - __name__, - globals()["__file__"], - _import_structure, - module_spec=__spec__, - ) - - for name, value in _dummy_objects.items(): - setattr(sys.modules[__name__], name, value) +# Copyright 2026 Pruna AI and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ..p_image_oss import PH3ImageBlocks, PH3ImageModularPipeline, PImageOSSBlocks, PImageOSSModularPipeline # noqa: F401 diff --git a/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py b/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py index 020201616522..1611f8e0fb30 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py +++ b/src/diffusers/modular_pipelines/p_h3_image/modular_blocks_p_h3_image.py @@ -11,458 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import torch - -from ..modular_pipeline import AutoPipelineBlocks, SequentialPipelineBlocks -from ..modular_pipeline_utils import OutputParam -from .before_denoise import ( - PH3ImageEditPrepareLatentsStep, - PH3ImageEditPrepareLayoutStep, - PH3ImagePrepareConditionLatentsStep, - PH3ImagePrepareLatentsStep, - PH3ImagePrepareLayoutStep, - PH3ImageSetTimestepsStep, -) -from .before_encoder import PH3ImageEditResizeStep -from .decoders import PH3ImageAfterDenoiseStep, PH3ImageDecodeStep -from .denoise import PH3ImageDenoiseStep -from .encoders import PH3ImageEditTextEncoderStep, PH3ImageEditVaeEncoderStep, PH3ImageTextEncoderStep -from .prompt_upsampler import PH3ImageAutoPromptUpsampleStep - - -# auto_docstring -class PH3ImageAutoBeforeEncodeStep(AutoPipelineBlocks): - """ - Image preparation block. - - `PH3ImageEditResizeStep` runs when `image` is provided (`image_conditioned`), resizing the image to edit onto - the canvas. - - a `text2image` request skips this block. - - Components: - image_processor (`VaeImageProcessor`) - - Configs: - reference_min_pixels (default: 1048576) reference_max_pixels (default: 8388608) canvas_min_pixels (default: - 4194304) vision_pixels (default: 1000000) - - Inputs: - image (`Image | list`, *optional*): - The image to edit, or a list of at most 8 images: the first is edited, the others are references. - height (`int`, *optional*): - Height of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for `text2image`, - which works best at 8 megapixels. An edit defaults to the first image's size, at least - `canvas_min_pixels`, and works best at 4 or 8 megapixels. - width (`int`, *optional*): - Width of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for `text2image`, - which works best at 8 megapixels. An edit defaults to the first image's size, at least - `canvas_min_pixels`, and works best at 4 or 8 megapixels. - - Outputs: - height (`int`): - Resolved height of the edited image in pixels. - width (`int`): - Resolved width of the edited image in pixels. - reference_images (`list`): - The images of the edit resized for the VAE. - vision_images (`list`): - The images of the edit resized for the text encoder. - """ - - model_name = "p-h3-image" - block_classes = [PH3ImageEditResizeStep] - block_names = ["image_conditioned"] - block_trigger_inputs = ["image"] - - @property - def description(self): - return ( - "Image preparation block.\n" - + " - `PH3ImageEditResizeStep` runs when `image` is provided (`image_conditioned`), resizing the image " - "to edit onto the canvas.\n" + " - a `text2image` request skips this block." - ) - - -# auto_docstring -class PH3ImageAutoTextEncoderStep(AutoPipelineBlocks): - """ - Text encoder block. - - `PH3ImageEditTextEncoderStep` runs when `image` is provided (`image_conditioned`). - - `PH3ImageTextEncoderStep` runs otherwise (`text2image`). - - Components: - text_encoder (`Qwen3VLForConditionalGeneration`) text_connector (`PH3ImageTextConnector`) tokenizer - (`Qwen2Tokenizer`) processor (`Qwen3VLProcessor`) - - Inputs: - prompt (`str`): - The prompt, or the edit instruction when `image` is passed. A single string. - vision_images (`list`, *optional*): - The images of the edit resized for the text encoder, the image to edit first. - - Outputs: - prompt_embeds (`Tensor`): - The conditioning MiniMax-H3's transformer reads, of shape `(1, num_text_tokens, 5120)`. - text_token_tags (`Tensor`): - The modality tag of every row of `prompt_embeds`; vision tokens are tagged as video. - """ - - model_name = "p-h3-image" - block_classes = [PH3ImageEditTextEncoderStep, PH3ImageTextEncoderStep] - block_names = ["image_conditioned", "text2image"] - block_trigger_inputs = ["image", None] - - @property - def description(self): - return ( - "Text encoder block.\n" - + " - `PH3ImageEditTextEncoderStep` runs when `image` is provided (`image_conditioned`).\n" - + " - `PH3ImageTextEncoderStep` runs otherwise (`text2image`)." - ) - - -# auto_docstring -class PH3ImageAutoVaeEncoderStep(AutoPipelineBlocks): - """ - VAE encoder block. - - `PH3ImageEditVaeEncoderStep` runs when `image` is provided (`image_conditioned`). - - a `text2image` request skips this block. - - Components: - vae (`AutoencoderKLPH3Image`) - - Inputs: - reference_images (`list`, *optional*): - The images of the edit resized for the VAE, the image to edit first. - - Outputs: - condition_latents (`list`): - The conditioning latents of the images of the edit, one `(1, latent_channels, 1, latent_height, - latent_width)` tensor per image. - """ - - model_name = "p-h3-image" - block_classes = [PH3ImageEditVaeEncoderStep] - block_names = ["image_conditioned"] - block_trigger_inputs = ["image"] - - @property - def description(self): - return ( - "VAE encoder block.\n" - + " - `PH3ImageEditVaeEncoderStep` runs when `image` is provided (`image_conditioned`).\n" - + " - a `text2image` request skips this block." - ) - - -# auto_docstring -class PH3ImageCoreDenoiseStep(SequentialPipelineBlocks): - """ - Core denoising workflow for `text2image`: builds the packed layout, draws the noise, runs the denoising loop and - unpacks the latents. - - Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer - (`MiniMaxH3Transformer3DModel`) - - Configs: - sample_sigmas (default: [1.0, 0.984753, 0.967905, 0.951688, 0.925372, 0.881923, 0.810083, 0.528718, 0.0]): - The checkpoint's sampling grid, used when `sigmas` is not passed. - - Inputs: - text_token_tags (`Tensor`): - The per-row modality tag of every row of `prompt_embeds`. - height (`int`, *optional*, defaults to 2912): - Height of the generated image in pixels, rounded to a multiple of 32. The model works best at 8 - megapixels. - width (`int`, *optional*, defaults to 2912): - Width of the generated image in pixels, rounded to a multiple of 32. The model works best at 8 - megapixels. - generator (`Generator`, *optional*): - Torch generator for deterministic generation. - latents (`Tensor`, *optional*): - Pre-generated image noise of shape `(1, latent_channels, 1, latent_height, latent_width)`, used instead - of the draw. - sigmas (`list`, *optional*): - A custom sampling grid, strictly decreasing and ending at 0. Defaults to the checkpoint's - `sample_sigmas`. - prompt_embeds (`Tensor`): - text embeddings used to guide the image generation. Can be generated from text_encoder step. - **denoiser_input_fields (`None`, *optional*): - The structural description of the packed sequence the transformer reads by name: `token_tags`, - `position_ids` and the three row-index tensors. - attention_kwargs (`dict`, *optional*): - Additional kwargs for attention processors. - - Outputs: - latents (`Tensor`): - The generated latents, of shape `(1, latent_channels, 1, latent_height, latent_width)`. - """ - - model_name = "p-h3-image" - block_classes = [ - PH3ImagePrepareLayoutStep, - PH3ImagePrepareLatentsStep, - PH3ImageSetTimestepsStep, - PH3ImageDenoiseStep, - PH3ImageAfterDenoiseStep, - ] - block_names = ["prepare_layout", "prepare_latents", "set_timesteps", "denoise", "after_denoise"] - - @property - def description(self): - return ( - "Core denoising workflow for `text2image`: builds the packed layout, draws the noise, runs the denoising " - "loop and unpacks the latents." - ) - - @property - def outputs(self): - return [ - OutputParam( - "latents", - type_hint=torch.Tensor, - description="The generated latents, of shape `(1, latent_channels, 1, latent_height, latent_width)`.", - ), - ] - - -# auto_docstring -class PH3ImageEditCoreDenoiseStep(SequentialPipelineBlocks): - """ - Core denoising workflow for `image_conditioned`: as for `text2image`, with the conditioning latents of the image to - edit packed in front of the generated rows. - - Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer - (`MiniMaxH3Transformer3DModel`) - - Configs: - sample_sigmas (default: [1.0, 0.984753, 0.967905, 0.951688, 0.925372, 0.881923, 0.810083, 0.528718, 0.0]): - The checkpoint's sampling grid, used when `sigmas` is not passed. - - Inputs: - text_token_tags (`Tensor`): - The per-row modality tag of every row of `prompt_embeds`. - condition_latents (`list`): - The conditioning latents of the images of the edit. Their shapes give the geometry of the reference - blocks. - height (`int`): - Height of the edited image in pixels. - width (`int`): - Width of the edited image in pixels. - generator (`Generator`, *optional*): - The generator of the request. The conditioning noise is drawn from it first, one draw per condition, - before the noise of the generated rows. - latents (`Tensor`, *optional*): - Pre-generated image noise of shape `(1, latent_channels, 1, latent_height, latent_width)`, used instead - of the draw. - sigmas (`list`, *optional*): - A custom sampling grid, strictly decreasing and ending at 0. Defaults to the checkpoint's - `sample_sigmas`. - prompt_embeds (`Tensor`): - text embeddings used to guide the image generation. Can be generated from text_encoder step. - **denoiser_input_fields (`None`, *optional*): - The structural description of the packed sequence the transformer reads by name: `token_tags`, - `position_ids` and the three row-index tensors. - attention_kwargs (`dict`, *optional*): - Additional kwargs for attention processors. - - Outputs: - latents (`Tensor`): - The generated latents, of shape `(1, latent_channels, 1, latent_height, latent_width)`. - """ - - model_name = "p-h3-image" - block_classes = [ - PH3ImageEditPrepareLayoutStep, - PH3ImagePrepareConditionLatentsStep, - PH3ImagePrepareLatentsStep, - PH3ImageEditPrepareLatentsStep, - PH3ImageSetTimestepsStep, - PH3ImageDenoiseStep, - PH3ImageAfterDenoiseStep, - ] - block_names = [ - "prepare_layout", - "prepare_condition_latents", - "prepare_latents", - "prepare_latents_image_conditioned", - "set_timesteps", - "denoise", - "after_denoise", - ] - - @property - def description(self): - return ( - "Core denoising workflow for `image_conditioned`: as for `text2image`, with the conditioning latents of the " - "image to edit packed in front of the generated rows." - ) - - @property - def outputs(self): - return [ - OutputParam( - "latents", - type_hint=torch.Tensor, - description="The generated latents, of shape `(1, latent_channels, 1, latent_height, latent_width)`.", - ), - ] - - -# auto_docstring -class PH3ImageAutoDenoiseStep(AutoPipelineBlocks): - """ - Denoise block. - - `PH3ImageEditCoreDenoiseStep` runs when `image` is provided (`image_conditioned`). - - `PH3ImageCoreDenoiseStep` runs otherwise (`text2image`). - - Components: - scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer - (`MiniMaxH3Transformer3DModel`) - - Configs: - sample_sigmas (default: [1.0, 0.984753, 0.967905, 0.951688, 0.925372, 0.881923, 0.810083, 0.528718, 0.0]): - The checkpoint's sampling grid, used when `sigmas` is not passed. - - Inputs: - text_token_tags (`Tensor`): - The per-row modality tag of every row of `prompt_embeds`. - condition_latents (`list`, *optional*): - The conditioning latents of the images of the edit. Their shapes give the geometry of the reference - blocks. - height (`int`, *optional*, defaults to None or 2912, depending on the workflow): - Height of the edited image in pixels. - width (`int`, *optional*, defaults to None or 2912, depending on the workflow): - Width of the edited image in pixels. - generator (`Generator`, *optional*): - The generator of the request. The conditioning noise is drawn from it first, one draw per condition, - before the noise of the generated rows. - latents (`Tensor`): - Pre-generated image noise of shape `(1, latent_channels, 1, latent_height, latent_width)`, used instead - of the draw. - sigmas (`list`, *optional*): - A custom sampling grid, strictly decreasing and ending at 0. Defaults to the checkpoint's - `sample_sigmas`. - prompt_embeds (`Tensor`): - text embeddings used to guide the image generation. Can be generated from text_encoder step. - **denoiser_input_fields (`None`, *optional*): - The structural description of the packed sequence the transformer reads by name: `token_tags`, - `position_ids` and the three row-index tensors. - attention_kwargs (`dict`, *optional*): - Additional kwargs for attention processors. - - Outputs: - latents (`Tensor`): - The generated latents, of shape `(1, latent_channels, 1, latent_height, latent_width)`. - """ - - model_name = "p-h3-image" - block_classes = [PH3ImageEditCoreDenoiseStep, PH3ImageCoreDenoiseStep] - block_names = ["image_conditioned", "text2image"] - block_trigger_inputs = ["image", None] - - @property - def description(self): - return ( - "Denoise block.\n" - + " - `PH3ImageEditCoreDenoiseStep` runs when `image` is provided (`image_conditioned`).\n" - + " - `PH3ImageCoreDenoiseStep` runs otherwise (`text2image`)." - ) - - -# auto_docstring -class PH3ImageBlocks(SequentialPipelineBlocks): - """ - Modular pipeline blocks for P-H3-Image: `text2image`, and instruction-based editing (`image_conditioned`) when - `image` is passed. `prompt` goes through the prompt upsampler first; pop the `prompt_upsampler` step to render it - as given. - - Supported workflows: - - `text2image`: requires `prompt` - - `image_conditioned`: requires `prompt`, `image` - - Components: - image_processor (`VaeImageProcessor`) prompt_upsampler (`Qwen3_5ForConditionalGeneration`): The prompt - upsampler. prompt_upsampler_processor (`AutoProcessor`): Its processor. text_encoder - (`Qwen3VLForConditionalGeneration`) text_connector (`PH3ImageTextConnector`) tokenizer (`Qwen2Tokenizer`) - processor (`Qwen3VLProcessor`) vae (`AutoencoderKLPH3Image`) scheduler (`MiniMaxH3Scheduler`) audio_scheduler - (`MiniMaxH3Scheduler`) transformer (`MiniMaxH3Transformer3DModel`) - - Configs: - reference_min_pixels (default: 1048576) reference_max_pixels (default: 8388608) canvas_min_pixels (default: - 4194304) vision_pixels (default: 1000000) sample_sigmas (default: [1.0, 0.984753, 0.967905, 0.951688, - 0.925372, 0.881923, 0.810083, 0.528718, 0.0]): The checkpoint's sampling grid, used when `sigmas` is not - passed. - - Inputs: - image (`Image | list`, *optional*): - The image to edit, or a list of at most 8 images: the first is edited, the others are references. - height (`int`, *optional*): - Height of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for `text2image`, - which works best at 8 megapixels. An edit defaults to the first image's size, at least - `canvas_min_pixels`, and works best at 4 or 8 megapixels. - width (`int`, *optional*): - Width of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for `text2image`, - which works best at 8 megapixels. An edit defaults to the first image's size, at least - `canvas_min_pixels`, and works best at 4 or 8 megapixels. - prompt (`str`): - The prompt, or the edit instruction when `image` is passed. A single string. - max_new_tokens (`int`, *optional*, defaults to 1536 or 1024, depending on the workflow): - The most tokens the upsampler may generate, its reasoning included. - vision_images (`list`, *optional*): - The images of the edit resized for the text encoder, the image to edit first. - reference_images (`list`, *optional*): - The images of the edit resized for the VAE, the image to edit first. - condition_latents (`list`, *optional*): - The conditioning latents of the images of the edit. Their shapes give the geometry of the reference - blocks. - generator (`Generator`, *optional*): - The generator of the request. The conditioning noise is drawn from it first, one draw per condition, - before the noise of the generated rows. - latents (`Tensor`): - Pre-generated image noise of shape `(1, latent_channels, 1, latent_height, latent_width)`, used instead - of the draw. - sigmas (`list`, *optional*): - A custom sampling grid, strictly decreasing and ending at 0. Defaults to the checkpoint's - `sample_sigmas`. - **denoiser_input_fields (`None`, *optional*): - The structural description of the packed sequence the transformer reads by name: `token_tags`, - `position_ids` and the three row-index tensors. - attention_kwargs (`dict`, *optional*): - Additional kwargs for attention processors. - output_type (`str`, *optional*, defaults to pil): - Output format: 'pil', 'np', 'pt'. - - Outputs: - images (`list`): - Generated images. - """ - - model_name = "p-h3-image" - block_classes = [ - PH3ImageAutoBeforeEncodeStep, - PH3ImageAutoPromptUpsampleStep, - PH3ImageAutoTextEncoderStep, - PH3ImageAutoVaeEncoderStep, - PH3ImageAutoDenoiseStep, - PH3ImageDecodeStep, - ] - block_names = ["before_encode", "prompt_upsampler", "text_encoder", "vae_encoder", "denoise", "decode"] - _workflow_map = { - "text2image": {"prompt": True}, - "image_conditioned": {"prompt": True, "image": True}, - } - - @property - def description(self): - return ( - "Modular pipeline blocks for P-H3-Image: `text2image`, and instruction-based editing (`image_conditioned`) " - "when `image` is passed. `prompt` goes through the prompt upsampler first; pop the `prompt_upsampler` step " - "to render it as given." - ) - - @property - def outputs(self): - return [OutputParam.template("images")] +from ..p_image_oss.modular_blocks_p_image_oss import PH3ImageBlocks, PImageOSSBlocks # noqa: F401 diff --git a/src/diffusers/modular_pipelines/p_h3_image/modular_pipeline.py b/src/diffusers/modular_pipelines/p_h3_image/modular_pipeline.py index 4973939a459d..46ecfddcc393 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/p_h3_image/modular_pipeline.py @@ -1,4 +1,4 @@ -# Copyright 2026 Pruna AI, The MiniMax and HuggingFace Teams. All rights reserved. +# Copyright 2026 Pruna AI and The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,113 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from ...utils import logging -from ..modular_pipeline import ModularPipeline - - -logger = logging.get_logger(__name__) # pylint: disable=invalid-name - - -class PH3ImageModularPipeline(ModularPipeline): - """ - A ModularPipeline for P-H3-Image, an image model based on MiniMax-H3, for text-to-image (`text2image`) and - instruction-based editing (`image_conditioned`). - - The image is a single latent frame, denoised in one packed sequence with two audio latents that are discarded. - - ```py - import torch - from diffusers import ComponentsManager, ModularPipeline - - manager = ComponentsManager() - manager.enable_auto_cpu_offload(device="cuda") - - pipe = ModularPipeline.from_pretrained("PrunaAI/P-H3-Image", components_manager=manager) - pipe.load_components(dtype=torch.bfloat16) - prompt = ( - "A German Expressionist oil painting of a Berlin street at night in the manner of Ernst Ludwig Kirchner, " - "with gaunt figures hurrying past a glowing gas lamp." - ) - image = pipe(prompt=prompt, height=3360, width=2496, output="images")[0] - ``` - """ - - default_blocks_name = "PH3ImageBlocks" - - @property - def vae_spatial_compression_ratio(self): - if getattr(self, "vae", None) is not None: - return self.vae.spatial_compression_ratio - return 16 - - @property - def vae_latent_channels(self): - if getattr(self, "vae", None) is not None: - return self.vae.config.latent_channels - return 24 - - @property - def audio_latent_channels(self): - r"""Channels of the audio latents, read off the transformer since there is no audio VAE.""" - if getattr(self, "transformer", None) is not None: - return self.transformer.config.audio_in_channels - return 32 - - @property - def num_audio_latents(self): - r"""Audio latents per channel denoised alongside the image.""" - return 2 - - @property - def patch_size(self): - if getattr(self, "transformer", None) is not None: - return tuple(self.transformer.config.patch_size) - return (1, 2, 2) - - @property - def canvas_multiple(self): - r"""What the generated height and width have to be a multiple of, 32 for the released checkpoint.""" - # A canvas has to survive the VAE's spatial compression and still be a whole number of patch rows wide, so - # the multiple is the product of the two. - return self.vae_spatial_compression_ratio * self.patch_size[2] - - @property - def audio_channels(self): - r"""Channels of the audio latents: MiniMax-H3's soundtrack is stereo, packed channel-major.""" - return 2 - - @property - def pixel_mean(self): - r"""Per-channel mean the VAE's input is normalized by, ImageNet's.""" - return (0.485, 0.456, 0.406) - - @property - def pixel_std(self): - r"""Per-channel standard deviation the VAE's input is normalized by, ImageNet's.""" - return (0.229, 0.224, 0.225) - - @property - def keyframe_encode_seed(self): - r"""Seed the conditioning posterior of an edit is sampled under, independently of the request's generator.""" - return 42 - - @property - def keyframe_noise_aug(self): - r"""The `t` the conditioning rows of an edit are held at: 0.999, just short of clean.""" - return 0.999 - - @property - def text_tag(self): - r"""The modality tag of a text row of the packed sequence.""" - return 1 - - @property - def video_tag(self): - r"""The modality tag of an image row of the packed sequence, which a vision block's rows also carry.""" - return 0 - - @property - def audio_tag(self): - r"""The modality tag of an audio row of the packed sequence.""" - return 2 +from ..p_image_oss.modular_pipeline import PH3ImageModularPipeline, PImageOSSModularPipeline # noqa: F401 diff --git a/src/diffusers/modular_pipelines/p_image_oss/__init__.py b/src/diffusers/modular_pipelines/p_image_oss/__init__.py new file mode 100644 index 000000000000..09f94acd22ee --- /dev/null +++ b/src/diffusers/modular_pipelines/p_image_oss/__init__.py @@ -0,0 +1,47 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["modular_blocks_p_image_oss"] = ["PH3ImageBlocks", "PImageOSSBlocks"] + _import_structure["modular_pipeline"] = ["PH3ImageModularPipeline", "PImageOSSModularPipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 + else: + from .modular_blocks_p_image_oss import PH3ImageBlocks, PImageOSSBlocks + from .modular_pipeline import PH3ImageModularPipeline, PImageOSSModularPipeline +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/src/diffusers/modular_pipelines/p_h3_image/before_denoise.py b/src/diffusers/modular_pipelines/p_image_oss/before_denoise.py similarity index 96% rename from src/diffusers/modular_pipelines/p_h3_image/before_denoise.py rename to src/diffusers/modular_pipelines/p_image_oss/before_denoise.py index 26bce6e30444..28cbad0b51ae 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/before_denoise.py +++ b/src/diffusers/modular_pipelines/p_image_oss/before_denoise.py @@ -21,7 +21,7 @@ from ..modular_pipeline import ModularPipelineBlocks, PipelineState from ..modular_pipeline_utils import ComponentSpec, ConfigSpec, InputParam, OutputParam from .before_encoder import round_canvas -from .modular_pipeline import PH3ImageModularPipeline +from .modular_pipeline import PImageOSSModularPipeline logger = logging.get_logger(__name__) # pylint: disable=invalid-name @@ -132,8 +132,8 @@ def _fill_audio_positions( ) -class PH3ImagePrepareLayoutStep(ModularPipelineBlocks): - model_name = "p-h3-image" +class PImageOSSPrepareLayoutStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -335,8 +335,8 @@ def build_packed_sequence( @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) device = components._execution_device @@ -380,8 +380,8 @@ def __call__( return components, state -class PH3ImageEditPrepareLayoutStep(ModularPipelineBlocks): - model_name = "p-h3-image" +class PImageOSSEditPrepareLayoutStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -557,8 +557,8 @@ def build_edit_packed_sequence( @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) device = components._execution_device @@ -604,9 +604,9 @@ def __call__( return components, state -# Copied from diffusers.modular_pipelines.minimax_h3.before_denoise.MiniMaxH3PrepareConditionLatentsStep with \bMiniMaxH3->PH3Image, minimax-h3->p-h3-image, PH3ImageScheduler->MiniMaxH3Scheduler -class PH3ImagePrepareConditionLatentsStep(ModularPipelineBlocks): - model_name = "p-h3-image" +# Copied from diffusers.modular_pipelines.minimax_h3.before_denoise.MiniMaxH3PrepareConditionLatentsStep with \bMiniMaxH3->PImageOSS, minimax-h3->p-image-oss, PImageOSSScheduler->MiniMaxH3Scheduler +class PImageOSSPrepareConditionLatentsStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -660,8 +660,8 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) device = components._execution_device patch_size = components.patch_size @@ -693,8 +693,8 @@ def __call__( return components, state -class PH3ImagePrepareLatentsStep(ModularPipelineBlocks): - model_name = "p-h3-image" +class PImageOSSPrepareLatentsStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -740,8 +740,8 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) device = components._execution_device @@ -761,9 +761,9 @@ def __call__( return components, state -# Copied from diffusers.modular_pipelines.minimax_h3.before_denoise.MiniMaxH3FL2VAPrepareLatentsStep with MiniMaxH3FL2VA->PH3ImageEdit, \bMiniMaxH3->PH3Image, minimax-h3->p-h3-image -class PH3ImageEditPrepareLatentsStep(ModularPipelineBlocks): - model_name = "p-h3-image" +# Copied from diffusers.modular_pipelines.minimax_h3.before_denoise.MiniMaxH3FL2VAPrepareLatentsStep with MiniMaxH3FL2VA->PImageOSSEdit, \bMiniMaxH3->PImageOSS, minimax-h3->p-image-oss +class PImageOSSEditPrepareLatentsStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -802,8 +802,8 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) block_state.latents = torch.cat([block_state.condition_rows, block_state.latents]) @@ -812,8 +812,8 @@ def __call__( return components, state -class PH3ImageSetTimestepsStep(ModularPipelineBlocks): - model_name = "p-h3-image" +class PImageOSSSetTimestepsStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -945,8 +945,8 @@ def build_row_timesteps( @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) device = components._execution_device diff --git a/src/diffusers/modular_pipelines/p_h3_image/before_encoder.py b/src/diffusers/modular_pipelines/p_image_oss/before_encoder.py similarity index 96% rename from src/diffusers/modular_pipelines/p_h3_image/before_encoder.py rename to src/diffusers/modular_pipelines/p_image_oss/before_encoder.py index 9a4cd36f543a..2c6326851dd8 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/before_encoder.py +++ b/src/diffusers/modular_pipelines/p_image_oss/before_encoder.py @@ -22,7 +22,7 @@ from ...utils import logging from ..modular_pipeline import ModularPipelineBlocks, PipelineState from ..modular_pipeline_utils import ComponentSpec, ConfigSpec, InputParam, OutputParam -from .modular_pipeline import PH3ImageModularPipeline +from .modular_pipeline import PImageOSSModularPipeline logger = logging.get_logger(__name__) # pylint: disable=invalid-name @@ -67,8 +67,8 @@ def _size_in_area_range(width: int, height: int, min_pixels: int, max_pixels: in return tuple(max(multiple, round(side * scale / multiple) * multiple) for side in (width, height)) -class PH3ImageEditResizeStep(ModularPipelineBlocks): - model_name = "p-h3-image" +class PImageOSSEditResizeStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -149,8 +149,8 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) images = block_state.image if isinstance(block_state.image, list) else [block_state.image] images = [image.convert("RGB") for image in images] diff --git a/src/diffusers/modular_pipelines/p_h3_image/decoders.py b/src/diffusers/modular_pipelines/p_image_oss/decoders.py similarity index 91% rename from src/diffusers/modular_pipelines/p_h3_image/decoders.py rename to src/diffusers/modular_pipelines/p_image_oss/decoders.py index 08e30927e9c9..7329bd1bf304 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/decoders.py +++ b/src/diffusers/modular_pipelines/p_image_oss/decoders.py @@ -16,19 +16,19 @@ from ...configuration_utils import FrozenDict from ...image_processor import VaeImageProcessor -from ...models import AutoencoderKLPH3Image +from ...models import AutoencoderKLPImageOSS from ...utils import logging from ..modular_pipeline import ModularPipelineBlocks, PipelineState from ..modular_pipeline_utils import ComponentSpec, InputParam, OutputParam -from .modular_pipeline import PH3ImageModularPipeline +from .modular_pipeline import PImageOSSModularPipeline logger = logging.get_logger(__name__) # pylint: disable=invalid-name -# Copied from diffusers.modular_pipelines.minimax_h3.decoders.MiniMaxH3AfterDenoiseStep with \bMiniMaxH3->PH3Image, minimax-h3->p-h3-image -class PH3ImageAfterDenoiseStep(ModularPipelineBlocks): - model_name = "p-h3-image" +# Copied from diffusers.modular_pipelines.minimax_h3.decoders.MiniMaxH3AfterDenoiseStep with \bMiniMaxH3->PImageOSS, minimax-h3->p-image-oss +class PImageOSSAfterDenoiseStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -98,8 +98,8 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) patch_t, patch_h, patch_w = components.patch_size channels = components.vae_latent_channels @@ -130,8 +130,8 @@ def __call__( return components, state -class PH3ImageDecodeStep(ModularPipelineBlocks): - model_name = "p-h3-image" +class PImageOSSDecodeStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -140,7 +140,7 @@ def description(self) -> str: @property def expected_components(self) -> list[ComponentSpec]: return [ - ComponentSpec("vae", AutoencoderKLPH3Image), + ComponentSpec("vae", AutoencoderKLPImageOSS), ComponentSpec( "image_processor", VaeImageProcessor, @@ -168,8 +168,8 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) device = components._execution_device diff --git a/src/diffusers/modular_pipelines/p_h3_image/denoise.py b/src/diffusers/modular_pipelines/p_image_oss/denoise.py similarity index 88% rename from src/diffusers/modular_pipelines/p_h3_image/denoise.py rename to src/diffusers/modular_pipelines/p_image_oss/denoise.py index 819e87cf4a5f..86d0cd5e9f1a 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/denoise.py +++ b/src/diffusers/modular_pipelines/p_image_oss/denoise.py @@ -26,15 +26,15 @@ PipelineState, ) from ..modular_pipeline_utils import ComponentSpec, InputParam, OutputParam -from .modular_pipeline import PH3ImageModularPipeline +from .modular_pipeline import PImageOSSModularPipeline logger = logging.get_logger(__name__) # pylint: disable=invalid-name -# Copied from diffusers.modular_pipelines.minimax_h3.denoise.MiniMaxH3LoopDenoiser with \bMiniMaxH3->PH3Image, minimax-h3->p-h3-image, PH3ImageTransformer3DModel->MiniMaxH3Transformer3DModel -class PH3ImageLoopDenoiser(ModularPipelineBlocks): - model_name = "p-h3-image" +# Copied from diffusers.modular_pipelines.minimax_h3.denoise.MiniMaxH3LoopDenoiser with \bMiniMaxH3->PImageOSS, minimax-h3->p-image-oss, PImageOSSTransformer3DModel->MiniMaxH3Transformer3DModel +class PImageOSSLoopDenoiser(ModularPipelineBlocks): + model_name = "p-image-oss" def __init__(self, transformer_name: str = "transformer"): r""" @@ -110,8 +110,8 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, block_state: BlockState, i: int, t: torch.Tensor - ) -> tuple[PH3ImageModularPipeline, BlockState]: + self, components: PImageOSSModularPipeline, block_state: BlockState, i: int, t: torch.Tensor + ) -> tuple[PImageOSSModularPipeline, BlockState]: transformer = getattr(components, self.transformer_name) unique_timesteps, timestep_indices = block_state.row_timestep_plan[i] # The layout tags its outputs `denoiser_input_fields`, and their names are the transformer's own parameter @@ -135,9 +135,9 @@ def __call__( return components, block_state -# Copied from diffusers.modular_pipelines.minimax_h3.denoise.MiniMaxH3LoopSchedulerStep with \bMiniMaxH3->PH3Image, minimax-h3->p-h3-image, PH3ImageScheduler->MiniMaxH3Scheduler -class PH3ImageLoopSchedulerStep(ModularPipelineBlocks): - model_name = "p-h3-image" +# Copied from diffusers.modular_pipelines.minimax_h3.denoise.MiniMaxH3LoopSchedulerStep with \bMiniMaxH3->PImageOSS, minimax-h3->p-image-oss, PImageOSSScheduler->MiniMaxH3Scheduler +class PImageOSSLoopSchedulerStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -217,8 +217,8 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, block_state: BlockState, i: int, t: torch.Tensor - ) -> tuple[PH3ImageModularPipeline, BlockState]: + self, components: PImageOSSModularPipeline, block_state: BlockState, i: int, t: torch.Tensor + ) -> tuple[PImageOSSModularPipeline, BlockState]: num_condition_video_rows = block_state.num_condition_video_rows num_condition_audio_rows = block_state.num_condition_audio_rows @@ -237,9 +237,9 @@ def __call__( return components, block_state -# Copied from diffusers.modular_pipelines.minimax_h3.denoise.MiniMaxH3DenoiseLoopWrapper with \bMiniMaxH3->PH3Image, minimax-h3->p-h3-image, PH3ImageScheduler->MiniMaxH3Scheduler -class PH3ImageDenoiseLoopWrapper(LoopSequentialPipelineBlocks): - model_name = "p-h3-image" +# Copied from diffusers.modular_pipelines.minimax_h3.denoise.MiniMaxH3DenoiseLoopWrapper with \bMiniMaxH3->PImageOSS, minimax-h3->p-image-oss, PImageOSSScheduler->MiniMaxH3Scheduler +class PImageOSSDenoiseLoopWrapper(LoopSequentialPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -260,8 +260,8 @@ def loop_inputs(self) -> list[InputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) with self.progress_bar(total=len(block_state.timesteps)) as progress_bar: for i, t in enumerate(block_state.timesteps): @@ -271,9 +271,9 @@ def __call__( return components, state -# Copied from diffusers.modular_pipelines.minimax_h3.denoise.MiniMaxH3DenoiseStep with \bMiniMaxH3->PH3Image -class PH3ImageDenoiseStep(PH3ImageDenoiseLoopWrapper): - block_classes = [PH3ImageLoopDenoiser, PH3ImageLoopSchedulerStep] +# Copied from diffusers.modular_pipelines.minimax_h3.denoise.MiniMaxH3DenoiseStep with \bMiniMaxH3->PImageOSS +class PImageOSSDenoiseStep(PImageOSSDenoiseLoopWrapper): + block_classes = [PImageOSSLoopDenoiser, PImageOSSLoopSchedulerStep] block_names = ["denoiser", "update"] @property diff --git a/src/diffusers/modular_pipelines/p_h3_image/encoders.py b/src/diffusers/modular_pipelines/p_image_oss/encoders.py similarity index 87% rename from src/diffusers/modular_pipelines/p_h3_image/encoders.py rename to src/diffusers/modular_pipelines/p_image_oss/encoders.py index b84b009810be..7dec55ddefc5 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/encoders.py +++ b/src/diffusers/modular_pipelines/p_image_oss/encoders.py @@ -17,17 +17,17 @@ import torch from transformers import Qwen2TokenizerFast, Qwen3VLForConditionalGeneration, Qwen3VLProcessor -from ...models import AutoencoderKLPH3Image, PH3ImageTextConnector +from ...models import AutoencoderKLPImageOSS, PImageOSSTextConnector from ...utils import logging from ..modular_pipeline import ModularPipelineBlocks, PipelineState from ..modular_pipeline_utils import ComponentSpec, InputParam, OutputParam -from .modular_pipeline import PH3ImageModularPipeline +from .modular_pipeline import PImageOSSModularPipeline logger = logging.get_logger(__name__) # pylint: disable=invalid-name -# Copied from diffusers.modular_pipelines.minimax_h3.encoders.encode_vae_condition with AutoencoderKLMiniMaxH3->AutoencoderKLPH3Image +# Copied from diffusers.modular_pipelines.minimax_h3.encoders.encode_vae_condition with AutoencoderKLMiniMaxH3->AutoencoderKLPImageOSS def encode_vae_condition( vae, pixels: torch.Tensor, pixel_mean: tuple, pixel_std: tuple, encode_seed: int = 42 ) -> torch.Tensor: @@ -41,7 +41,7 @@ def encode_vae_condition( what turns `17 * n + 5` frames into `5 * n + 2` latent frames. Args: - vae (`AutoencoderKLPH3Image`): The video VAE. + vae (`AutoencoderKLPImageOSS`): The video VAE. pixels (`torch.Tensor` of shape `(1, 3, num_frames, height, width)`): The condition's `uint8` pixels, on the VAE's device. `num_frames` is 1 for a keyframe or an image reference. @@ -65,7 +65,7 @@ def encode_vae_condition( return (latents - latents_mean) / latents_std -def get_p_h3_image_prompt_embeds( +def get_p_image_oss_prompt_embeds( text_encoder, text_connector, processor, @@ -75,12 +75,12 @@ def get_p_h3_image_prompt_embeds( dtype: torch.dtype | None = None, ) -> torch.Tensor: r""" - Encode one tokenized P-H3-Image presentation: the hidden states of the text encoder, mapped to the transformer's + Encode one tokenized P-Image-OSS presentation: the hidden states of the text encoder, mapped to the transformer's text conditioning by the text connector. Args: text_encoder (`Qwen3VLForConditionalGeneration`): The text encoder. - text_connector (`PH3ImageTextConnector`): Maps the text encoder's hidden states to the conditioning. + text_connector (`PImageOSSTextConnector`): Maps the text encoder's hidden states to the conditioning. processor (`Qwen3VLProcessor`): The text encoder's processor, used for the token type ids. token_ids (`list[int]`): The tokenized presentation. vision_inputs (`dict`, *optional*): @@ -111,8 +111,8 @@ def get_p_h3_image_prompt_embeds( return text_connector(outputs.hidden_states).to(device=device, dtype=dtype) -class PH3ImageTextEncoderStep(ModularPipelineBlocks): - model_name = "p-h3-image" +class PImageOSSTextEncoderStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -122,7 +122,7 @@ def description(self) -> str: def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("text_encoder", Qwen3VLForConditionalGeneration), - ComponentSpec("text_connector", PH3ImageTextConnector), + ComponentSpec("text_connector", PImageOSSTextConnector), ComponentSpec("tokenizer", Qwen2TokenizerFast), ComponentSpec("processor", Qwen3VLProcessor), ] @@ -149,18 +149,18 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) if not isinstance(block_state.prompt, str): raise ValueError( - "P-H3-Image packs one request into one sequence, so `prompt` must be a single string, got " + "P-Image-OSS packs one request into one sequence, so `prompt` must be a single string, got " f"{type(block_state.prompt)}." ) token_ids = components.tokenizer(block_state.prompt, add_special_tokens=False)["input_ids"] - block_state.prompt_embeds = get_p_h3_image_prompt_embeds( + block_state.prompt_embeds = get_p_image_oss_prompt_embeds( components.text_encoder, components.text_connector, components.processor, @@ -174,8 +174,8 @@ def __call__( return components, state -class PH3ImageEditTextEncoderStep(ModularPipelineBlocks): - model_name = "p-h3-image" +class PImageOSSEditTextEncoderStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -189,7 +189,7 @@ def description(self) -> str: def expected_components(self) -> list[ComponentSpec]: return [ ComponentSpec("text_encoder", Qwen3VLForConditionalGeneration), - ComponentSpec("text_connector", PH3ImageTextConnector), + ComponentSpec("text_connector", PImageOSSTextConnector), ComponentSpec("tokenizer", Qwen2TokenizerFast), ComponentSpec("processor", Qwen3VLProcessor), ] @@ -224,12 +224,12 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) if not isinstance(block_state.prompt, str): raise ValueError( - "P-H3-Image packs one request into one sequence, so `prompt` must be a single string, got " + "P-Image-OSS packs one request into one sequence, so `prompt` must be a single string, got " f"{type(block_state.prompt)}." ) tokenizer, processor = components.tokenizer, components.processor @@ -254,7 +254,7 @@ def __call__( token_ids += prompt_ids token_tags += [text_tag] * len(prompt_ids) - block_state.prompt_embeds = get_p_h3_image_prompt_embeds( + block_state.prompt_embeds = get_p_image_oss_prompt_embeds( components.text_encoder, components.text_connector, processor, @@ -269,8 +269,8 @@ def __call__( return components, state -class PH3ImageEditVaeEncoderStep(ModularPipelineBlocks): - model_name = "p-h3-image" +class PImageOSSEditVaeEncoderStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: @@ -281,7 +281,7 @@ def description(self) -> str: @property def expected_components(self) -> list[ComponentSpec]: - return [ComponentSpec("vae", AutoencoderKLPH3Image)] + return [ComponentSpec("vae", AutoencoderKLPImageOSS)] @property def inputs(self) -> list[InputParam]: @@ -309,8 +309,8 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) device = components._execution_device diff --git a/src/diffusers/modular_pipelines/p_image_oss/modular_blocks_p_image_oss.py b/src/diffusers/modular_pipelines/p_image_oss/modular_blocks_p_image_oss.py new file mode 100644 index 000000000000..1e687de4383e --- /dev/null +++ b/src/diffusers/modular_pipelines/p_image_oss/modular_blocks_p_image_oss.py @@ -0,0 +1,472 @@ +# Copyright 2026 Pruna AI and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from ..modular_pipeline import AutoPipelineBlocks, SequentialPipelineBlocks +from ..modular_pipeline_utils import OutputParam +from .before_denoise import ( + PImageOSSEditPrepareLatentsStep, + PImageOSSEditPrepareLayoutStep, + PImageOSSPrepareConditionLatentsStep, + PImageOSSPrepareLatentsStep, + PImageOSSPrepareLayoutStep, + PImageOSSSetTimestepsStep, +) +from .before_encoder import PImageOSSEditResizeStep +from .decoders import PImageOSSAfterDenoiseStep, PImageOSSDecodeStep +from .denoise import PImageOSSDenoiseStep +from .encoders import PImageOSSEditTextEncoderStep, PImageOSSEditVaeEncoderStep, PImageOSSTextEncoderStep +from .prompt_upsampler import PImageOSSAutoPromptUpsampleStep + + +# auto_docstring +class PImageOSSAutoBeforeEncodeStep(AutoPipelineBlocks): + """ + Image preparation block. + - `PImageOSSEditResizeStep` runs when `image` is provided (`image_conditioned`), resizing the image to edit onto + the canvas. + - a `text2image` request skips this block. + + Components: + image_processor (`VaeImageProcessor`) + + Configs: + reference_min_pixels (default: 1048576) reference_max_pixels (default: 8388608) canvas_min_pixels (default: + 4194304) vision_pixels (default: 1000000) + + Inputs: + image (`Image | list`, *optional*): + The image to edit, or a list of at most 8 images: the first is edited, the others are references. + height (`int`, *optional*): + Height of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for `text2image`, + which works best at 8 megapixels. An edit defaults to the first image's size, at least + `canvas_min_pixels`, and works best at 4 or 8 megapixels. + width (`int`, *optional*): + Width of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for `text2image`, + which works best at 8 megapixels. An edit defaults to the first image's size, at least + `canvas_min_pixels`, and works best at 4 or 8 megapixels. + + Outputs: + height (`int`): + Resolved height of the edited image in pixels. + width (`int`): + Resolved width of the edited image in pixels. + reference_images (`list`): + The images of the edit resized for the VAE. + vision_images (`list`): + The images of the edit resized for the text encoder. + """ + + model_name = "p-image-oss" + block_classes = [PImageOSSEditResizeStep] + block_names = ["image_conditioned"] + block_trigger_inputs = ["image"] + + @property + def description(self): + return ( + "Image preparation block.\n" + + " - `PImageOSSEditResizeStep` runs when `image` is provided (`image_conditioned`), resizing the image " + "to edit onto the canvas.\n" + " - a `text2image` request skips this block." + ) + + +# auto_docstring +class PImageOSSAutoTextEncoderStep(AutoPipelineBlocks): + """ + Text encoder block. + - `PImageOSSEditTextEncoderStep` runs when `image` is provided (`image_conditioned`). + - `PImageOSSTextEncoderStep` runs otherwise (`text2image`). + + Components: + text_encoder (`Qwen3VLForConditionalGeneration`) text_connector (`PImageOSSTextConnector`) tokenizer + (`Qwen2Tokenizer`) processor (`Qwen3VLProcessor`) + + Inputs: + prompt (`str`): + The prompt, or the edit instruction when `image` is passed. A single string. + vision_images (`list`, *optional*): + The images of the edit resized for the text encoder, the image to edit first. + + Outputs: + prompt_embeds (`Tensor`): + The conditioning MiniMax-H3's transformer reads, of shape `(1, num_text_tokens, 5120)`. + text_token_tags (`Tensor`): + The modality tag of every row of `prompt_embeds`; vision tokens are tagged as video. + """ + + model_name = "p-image-oss" + block_classes = [PImageOSSEditTextEncoderStep, PImageOSSTextEncoderStep] + block_names = ["image_conditioned", "text2image"] + block_trigger_inputs = ["image", None] + + @property + def description(self): + return ( + "Text encoder block.\n" + + " - `PImageOSSEditTextEncoderStep` runs when `image` is provided (`image_conditioned`).\n" + + " - `PImageOSSTextEncoderStep` runs otherwise (`text2image`)." + ) + + +# auto_docstring +class PImageOSSAutoVaeEncoderStep(AutoPipelineBlocks): + """ + VAE encoder block. + - `PImageOSSEditVaeEncoderStep` runs when `image` is provided (`image_conditioned`). + - a `text2image` request skips this block. + + Components: + vae (`AutoencoderKLPImageOSS`) + + Inputs: + reference_images (`list`, *optional*): + The images of the edit resized for the VAE, the image to edit first. + + Outputs: + condition_latents (`list`): + The conditioning latents of the images of the edit, one `(1, latent_channels, 1, latent_height, + latent_width)` tensor per image. + """ + + model_name = "p-image-oss" + block_classes = [PImageOSSEditVaeEncoderStep] + block_names = ["image_conditioned"] + block_trigger_inputs = ["image"] + + @property + def description(self): + return ( + "VAE encoder block.\n" + + " - `PImageOSSEditVaeEncoderStep` runs when `image` is provided (`image_conditioned`).\n" + + " - a `text2image` request skips this block." + ) + + +# auto_docstring +class PImageOSSCoreDenoiseStep(SequentialPipelineBlocks): + """ + Core denoising workflow for `text2image`: builds the packed layout, draws the noise, runs the denoising loop and + unpacks the latents. + + Components: + scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer + (`MiniMaxH3Transformer3DModel`) + + Configs: + sample_sigmas (default: [1.0, 0.984753, 0.967905, 0.951688, 0.925372, 0.881923, 0.810083, 0.528718, 0.0]): + The checkpoint's sampling grid, used when `sigmas` is not passed. + + Inputs: + text_token_tags (`Tensor`): + The per-row modality tag of every row of `prompt_embeds`. + height (`int`, *optional*, defaults to 2912): + Height of the generated image in pixels, rounded to a multiple of 32. The model works best at 8 + megapixels. + width (`int`, *optional*, defaults to 2912): + Width of the generated image in pixels, rounded to a multiple of 32. The model works best at 8 + megapixels. + generator (`Generator`, *optional*): + Torch generator for deterministic generation. + latents (`Tensor`, *optional*): + Pre-generated image noise of shape `(1, latent_channels, 1, latent_height, latent_width)`, used instead + of the draw. + sigmas (`list`, *optional*): + A custom sampling grid, strictly decreasing and ending at 0. Defaults to the checkpoint's + `sample_sigmas`. + prompt_embeds (`Tensor`): + text embeddings used to guide the image generation. Can be generated from text_encoder step. + **denoiser_input_fields (`None`, *optional*): + The structural description of the packed sequence the transformer reads by name: `token_tags`, + `position_ids` and the three row-index tensors. + attention_kwargs (`dict`, *optional*): + Additional kwargs for attention processors. + + Outputs: + latents (`Tensor`): + The generated latents, of shape `(1, latent_channels, 1, latent_height, latent_width)`. + """ + + model_name = "p-image-oss" + block_classes = [ + PImageOSSPrepareLayoutStep, + PImageOSSPrepareLatentsStep, + PImageOSSSetTimestepsStep, + PImageOSSDenoiseStep, + PImageOSSAfterDenoiseStep, + ] + block_names = ["prepare_layout", "prepare_latents", "set_timesteps", "denoise", "after_denoise"] + + @property + def description(self): + return ( + "Core denoising workflow for `text2image`: builds the packed layout, draws the noise, runs the denoising " + "loop and unpacks the latents." + ) + + @property + def outputs(self): + return [ + OutputParam( + "latents", + type_hint=torch.Tensor, + description="The generated latents, of shape `(1, latent_channels, 1, latent_height, latent_width)`.", + ), + ] + + +# auto_docstring +class PImageOSSEditCoreDenoiseStep(SequentialPipelineBlocks): + """ + Core denoising workflow for `image_conditioned`: as for `text2image`, with the conditioning latents of the image to + edit packed in front of the generated rows. + + Components: + scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer + (`MiniMaxH3Transformer3DModel`) + + Configs: + sample_sigmas (default: [1.0, 0.984753, 0.967905, 0.951688, 0.925372, 0.881923, 0.810083, 0.528718, 0.0]): + The checkpoint's sampling grid, used when `sigmas` is not passed. + + Inputs: + text_token_tags (`Tensor`): + The per-row modality tag of every row of `prompt_embeds`. + condition_latents (`list`): + The conditioning latents of the images of the edit. Their shapes give the geometry of the reference + blocks. + height (`int`): + Height of the edited image in pixels. + width (`int`): + Width of the edited image in pixels. + generator (`Generator`, *optional*): + The generator of the request. The conditioning noise is drawn from it first, one draw per condition, + before the noise of the generated rows. + latents (`Tensor`, *optional*): + Pre-generated image noise of shape `(1, latent_channels, 1, latent_height, latent_width)`, used instead + of the draw. + sigmas (`list`, *optional*): + A custom sampling grid, strictly decreasing and ending at 0. Defaults to the checkpoint's + `sample_sigmas`. + prompt_embeds (`Tensor`): + text embeddings used to guide the image generation. Can be generated from text_encoder step. + **denoiser_input_fields (`None`, *optional*): + The structural description of the packed sequence the transformer reads by name: `token_tags`, + `position_ids` and the three row-index tensors. + attention_kwargs (`dict`, *optional*): + Additional kwargs for attention processors. + + Outputs: + latents (`Tensor`): + The generated latents, of shape `(1, latent_channels, 1, latent_height, latent_width)`. + """ + + model_name = "p-image-oss" + block_classes = [ + PImageOSSEditPrepareLayoutStep, + PImageOSSPrepareConditionLatentsStep, + PImageOSSPrepareLatentsStep, + PImageOSSEditPrepareLatentsStep, + PImageOSSSetTimestepsStep, + PImageOSSDenoiseStep, + PImageOSSAfterDenoiseStep, + ] + block_names = [ + "prepare_layout", + "prepare_condition_latents", + "prepare_latents", + "prepare_latents_image_conditioned", + "set_timesteps", + "denoise", + "after_denoise", + ] + + @property + def description(self): + return ( + "Core denoising workflow for `image_conditioned`: as for `text2image`, with the conditioning latents of the " + "image to edit packed in front of the generated rows." + ) + + @property + def outputs(self): + return [ + OutputParam( + "latents", + type_hint=torch.Tensor, + description="The generated latents, of shape `(1, latent_channels, 1, latent_height, latent_width)`.", + ), + ] + + +# auto_docstring +class PImageOSSAutoDenoiseStep(AutoPipelineBlocks): + """ + Denoise block. + - `PImageOSSEditCoreDenoiseStep` runs when `image` is provided (`image_conditioned`). + - `PImageOSSCoreDenoiseStep` runs otherwise (`text2image`). + + Components: + scheduler (`MiniMaxH3Scheduler`) audio_scheduler (`MiniMaxH3Scheduler`) transformer + (`MiniMaxH3Transformer3DModel`) + + Configs: + sample_sigmas (default: [1.0, 0.984753, 0.967905, 0.951688, 0.925372, 0.881923, 0.810083, 0.528718, 0.0]): + The checkpoint's sampling grid, used when `sigmas` is not passed. + + Inputs: + text_token_tags (`Tensor`): + The per-row modality tag of every row of `prompt_embeds`. + condition_latents (`list`, *optional*): + The conditioning latents of the images of the edit. Their shapes give the geometry of the reference + blocks. + height (`int`, *optional*, defaults to None or 2912, depending on the workflow): + Height of the edited image in pixels. + width (`int`, *optional*, defaults to None or 2912, depending on the workflow): + Width of the edited image in pixels. + generator (`Generator`, *optional*): + The generator of the request. The conditioning noise is drawn from it first, one draw per condition, + before the noise of the generated rows. + latents (`Tensor`): + Pre-generated image noise of shape `(1, latent_channels, 1, latent_height, latent_width)`, used instead + of the draw. + sigmas (`list`, *optional*): + A custom sampling grid, strictly decreasing and ending at 0. Defaults to the checkpoint's + `sample_sigmas`. + prompt_embeds (`Tensor`): + text embeddings used to guide the image generation. Can be generated from text_encoder step. + **denoiser_input_fields (`None`, *optional*): + The structural description of the packed sequence the transformer reads by name: `token_tags`, + `position_ids` and the three row-index tensors. + attention_kwargs (`dict`, *optional*): + Additional kwargs for attention processors. + + Outputs: + latents (`Tensor`): + The generated latents, of shape `(1, latent_channels, 1, latent_height, latent_width)`. + """ + + model_name = "p-image-oss" + block_classes = [PImageOSSEditCoreDenoiseStep, PImageOSSCoreDenoiseStep] + block_names = ["image_conditioned", "text2image"] + block_trigger_inputs = ["image", None] + + @property + def description(self): + return ( + "Denoise block.\n" + + " - `PImageOSSEditCoreDenoiseStep` runs when `image` is provided (`image_conditioned`).\n" + + " - `PImageOSSCoreDenoiseStep` runs otherwise (`text2image`)." + ) + + +# auto_docstring +class PImageOSSBlocks(SequentialPipelineBlocks): + """ + Modular pipeline blocks for P-Image-OSS: `text2image`, and instruction-based editing (`image_conditioned`) when + `image` is passed. `prompt` goes through the prompt upsampler first; pop the `prompt_upsampler` step to render it + as given. + + Supported workflows: + - `text2image`: requires `prompt` + - `image_conditioned`: requires `prompt`, `image` + + Components: + image_processor (`VaeImageProcessor`) prompt_upsampler (`Qwen3_5ForConditionalGeneration`): The prompt + upsampler. prompt_upsampler_processor (`AutoProcessor`): Its processor. text_encoder + (`Qwen3VLForConditionalGeneration`) text_connector (`PImageOSSTextConnector`) tokenizer (`Qwen2Tokenizer`) + processor (`Qwen3VLProcessor`) vae (`AutoencoderKLPImageOSS`) scheduler (`MiniMaxH3Scheduler`) + audio_scheduler (`MiniMaxH3Scheduler`) transformer (`MiniMaxH3Transformer3DModel`) + + Configs: + reference_min_pixels (default: 1048576) reference_max_pixels (default: 8388608) canvas_min_pixels (default: + 4194304) vision_pixels (default: 1000000) sample_sigmas (default: [1.0, 0.984753, 0.967905, 0.951688, + 0.925372, 0.881923, 0.810083, 0.528718, 0.0]): The checkpoint's sampling grid, used when `sigmas` is not + passed. + + Inputs: + image (`Image | list`, *optional*): + The image to edit, or a list of at most 8 images: the first is edited, the others are references. + height (`int`, *optional*): + Height of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for `text2image`, + which works best at 8 megapixels. An edit defaults to the first image's size, at least + `canvas_min_pixels`, and works best at 4 or 8 megapixels. + width (`int`, *optional*): + Width of the generated image in pixels, rounded to a multiple of 32. Defaults to 2912 for `text2image`, + which works best at 8 megapixels. An edit defaults to the first image's size, at least + `canvas_min_pixels`, and works best at 4 or 8 megapixels. + prompt (`str`): + The prompt, or the edit instruction when `image` is passed. A single string. + max_new_tokens (`int`, *optional*, defaults to 1536 or 1024, depending on the workflow): + The most tokens the upsampler may generate, its reasoning included. + vision_images (`list`, *optional*): + The images of the edit resized for the text encoder, the image to edit first. + reference_images (`list`, *optional*): + The images of the edit resized for the VAE, the image to edit first. + condition_latents (`list`, *optional*): + The conditioning latents of the images of the edit. Their shapes give the geometry of the reference + blocks. + generator (`Generator`, *optional*): + The generator of the request. The conditioning noise is drawn from it first, one draw per condition, + before the noise of the generated rows. + latents (`Tensor`): + Pre-generated image noise of shape `(1, latent_channels, 1, latent_height, latent_width)`, used instead + of the draw. + sigmas (`list`, *optional*): + A custom sampling grid, strictly decreasing and ending at 0. Defaults to the checkpoint's + `sample_sigmas`. + **denoiser_input_fields (`None`, *optional*): + The structural description of the packed sequence the transformer reads by name: `token_tags`, + `position_ids` and the three row-index tensors. + attention_kwargs (`dict`, *optional*): + Additional kwargs for attention processors. + output_type (`str`, *optional*, defaults to pil): + Output format: 'pil', 'np', 'pt'. + + Outputs: + images (`list`): + Generated images. + """ + + model_name = "p-image-oss" + block_classes = [ + PImageOSSAutoBeforeEncodeStep, + PImageOSSAutoPromptUpsampleStep, + PImageOSSAutoTextEncoderStep, + PImageOSSAutoVaeEncoderStep, + PImageOSSAutoDenoiseStep, + PImageOSSDecodeStep, + ] + block_names = ["before_encode", "prompt_upsampler", "text_encoder", "vae_encoder", "denoise", "decode"] + _workflow_map = { + "text2image": {"prompt": True}, + "image_conditioned": {"prompt": True, "image": True}, + } + + @property + def description(self): + return ( + "Modular pipeline blocks for P-Image-OSS: `text2image`, and instruction-based editing (`image_conditioned`) " + "when `image` is passed. `prompt` goes through the prompt upsampler first; pop the `prompt_upsampler` step " + "to render it as given." + ) + + @property + def outputs(self): + return [OutputParam.template("images")] + + +# Deprecated alias for the former P-H3-Image name, kept so existing code and Hub configs keep loading. +PH3ImageBlocks = PImageOSSBlocks diff --git a/src/diffusers/modular_pipelines/p_image_oss/modular_pipeline.py b/src/diffusers/modular_pipelines/p_image_oss/modular_pipeline.py new file mode 100644 index 000000000000..a7287d440ada --- /dev/null +++ b/src/diffusers/modular_pipelines/p_image_oss/modular_pipeline.py @@ -0,0 +1,127 @@ +# Copyright 2026 Pruna AI, The MiniMax and HuggingFace Teams. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ...utils import logging +from ..modular_pipeline import ModularPipeline + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class PImageOSSModularPipeline(ModularPipeline): + """ + A ModularPipeline for P-Image-OSS, an image model based on MiniMax-H3, for text-to-image (`text2image`) and + instruction-based editing (`image_conditioned`). + + The image is a single latent frame, denoised in one packed sequence with two audio latents that are discarded. + + ```py + import torch + from diffusers import ComponentsManager, ModularPipeline + + manager = ComponentsManager() + manager.enable_auto_cpu_offload(device="cuda") + + pipe = ModularPipeline.from_pretrained("PrunaAI/P-Image-OSS", components_manager=manager) + pipe.load_components(dtype=torch.bfloat16) + prompt = ( + "A German Expressionist oil painting of a Berlin street at night in the manner of Ernst Ludwig Kirchner, " + "with gaunt figures hurrying past a glowing gas lamp." + ) + image = pipe(prompt=prompt, height=3360, width=2496, output="images")[0] + ``` + """ + + default_blocks_name = "PImageOSSBlocks" + + @property + def vae_spatial_compression_ratio(self): + if getattr(self, "vae", None) is not None: + return self.vae.spatial_compression_ratio + return 16 + + @property + def vae_latent_channels(self): + if getattr(self, "vae", None) is not None: + return self.vae.config.latent_channels + return 24 + + @property + def audio_latent_channels(self): + r"""Channels of the audio latents, read off the transformer since there is no audio VAE.""" + if getattr(self, "transformer", None) is not None: + return self.transformer.config.audio_in_channels + return 32 + + @property + def num_audio_latents(self): + r"""Audio latents per channel denoised alongside the image.""" + return 2 + + @property + def patch_size(self): + if getattr(self, "transformer", None) is not None: + return tuple(self.transformer.config.patch_size) + return (1, 2, 2) + + @property + def canvas_multiple(self): + r"""What the generated height and width have to be a multiple of, 32 for the released checkpoint.""" + # A canvas has to survive the VAE's spatial compression and still be a whole number of patch rows wide, so + # the multiple is the product of the two. + return self.vae_spatial_compression_ratio * self.patch_size[2] + + @property + def audio_channels(self): + r"""Channels of the audio latents: MiniMax-H3's soundtrack is stereo, packed channel-major.""" + return 2 + + @property + def pixel_mean(self): + r"""Per-channel mean the VAE's input is normalized by, ImageNet's.""" + return (0.485, 0.456, 0.406) + + @property + def pixel_std(self): + r"""Per-channel standard deviation the VAE's input is normalized by, ImageNet's.""" + return (0.229, 0.224, 0.225) + + @property + def keyframe_encode_seed(self): + r"""Seed the conditioning posterior of an edit is sampled under, independently of the request's generator.""" + return 42 + + @property + def keyframe_noise_aug(self): + r"""The `t` the conditioning rows of an edit are held at: 0.999, just short of clean.""" + return 0.999 + + @property + def text_tag(self): + r"""The modality tag of a text row of the packed sequence.""" + return 1 + + @property + def video_tag(self): + r"""The modality tag of an image row of the packed sequence, which a vision block's rows also carry.""" + return 0 + + @property + def audio_tag(self): + r"""The modality tag of an audio row of the packed sequence.""" + return 2 + + +# Deprecated alias for the former P-H3-Image name, kept so existing code and Hub configs keep loading. +PH3ImageModularPipeline = PImageOSSModularPipeline diff --git a/src/diffusers/modular_pipelines/p_h3_image/prompt_upsampler.py b/src/diffusers/modular_pipelines/p_image_oss/prompt_upsampler.py similarity index 88% rename from src/diffusers/modular_pipelines/p_h3_image/prompt_upsampler.py rename to src/diffusers/modular_pipelines/p_image_oss/prompt_upsampler.py index 60302b9d4439..6d9408a5348f 100644 --- a/src/diffusers/modular_pipelines/p_h3_image/prompt_upsampler.py +++ b/src/diffusers/modular_pipelines/p_image_oss/prompt_upsampler.py @@ -22,7 +22,7 @@ from ..modular_pipeline import AutoPipelineBlocks, ModularPipelineBlocks, PipelineState from ..modular_pipeline_utils import ComponentSpec, InputParam, OutputParam from .before_encoder import MAX_EDIT_IMAGES, _size_at_area -from .modular_pipeline import PH3ImageModularPipeline +from .modular_pipeline import PImageOSSModularPipeline ASPECT_RATIOS = {"1:1": 1.0, "4:3": 4 / 3, "3:4": 3 / 4, "3:2": 3 / 2, "2:3": 2 / 3, "16:9": 16 / 9, "9:16": 9 / 16} @@ -52,7 +52,7 @@ def aspect_ratio_label(width: int, height: int) -> str: def is_edit_prompt(prompt: str) -> bool: r""" - Whether an edit prompt has one of the two forms P-H3-Image edits from. + Whether an edit prompt has one of the two forms P-Image-OSS edits from. A brief is the six sections of `EDIT_SECTIONS` in order, with `[keyframe completion]` once and every line of `retention_analysis` naming a `` or `` and how it is kept. An instruct prompt is one @@ -89,7 +89,7 @@ def upsample_prompt( max_new_tokens: int = T2I_MAX_NEW_TOKENS, ) -> str: """ - The prompt P-H3-Image renders for `request`: a text-to-image prompt for `aspect_ratio`, or an edit prompt written + The prompt P-Image-OSS renders for `request`: a text-to-image prompt for `aspect_ratio`, or an edit prompt written from the prepared `images` (the image to edit first). One greedy generation of at most `max_new_tokens`; the upsampler reasons first and the prompt is what follows ``. """ @@ -135,13 +135,13 @@ def upsample_prompt( return prompt -class PH3ImageText2ImagePromptUpsampleStep(ModularPipelineBlocks): - model_name = "p-h3-image" +class PImageOSSText2ImagePromptUpsampleStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: return ( - "Expands a text-to-image request into the full prompt P-H3-Image is meant to render, written by the " + "Expands a text-to-image request into the full prompt P-Image-OSS is meant to render, written by the " "prompt upsampler for the aspect ratio of `height` and `width`." ) @@ -174,8 +174,8 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) block_state.prompt = upsample_prompt( components.prompt_upsampler, @@ -188,13 +188,13 @@ def __call__( return components, state -class PH3ImageEditPromptUpsampleStep(ModularPipelineBlocks): - model_name = "p-h3-image" +class PImageOSSEditPromptUpsampleStep(ModularPipelineBlocks): + model_name = "p-image-oss" @property def description(self) -> str: return ( - "Expands an edit instruction into the edit prompt P-H3-Image is meant to render, written by the prompt " + "Expands an edit instruction into the edit prompt P-Image-OSS is meant to render, written by the prompt " "upsampler from the images of the edit (`image`): a brief of what to keep and what to change." ) @@ -230,8 +230,8 @@ def intermediate_outputs(self) -> list[OutputParam]: @torch.no_grad() def __call__( - self, components: PH3ImageModularPipeline, state: PipelineState - ) -> tuple[PH3ImageModularPipeline, PipelineState]: + self, components: PImageOSSModularPipeline, state: PipelineState + ) -> tuple[PImageOSSModularPipeline, PipelineState]: block_state = self.get_block_state(state) images = block_state.image if isinstance(block_state.image, list) else [block_state.image] if len(images) > MAX_EDIT_IMAGES: @@ -248,13 +248,13 @@ def __call__( # auto_docstring -class PH3ImageAutoPromptUpsampleStep(AutoPipelineBlocks): +class PImageOSSAutoPromptUpsampleStep(AutoPipelineBlocks): """ Prompt upsampling step: - expands `prompt` into the full prompt P-H3-Image is meant to render, with the prompt upsampler (based on + expands `prompt` into the full prompt P-Image-OSS is meant to render, with the prompt upsampler (based on Qwen3.5-9B). The task follows the inputs, like the rest of the pipeline: - - `PH3ImageEditPromptUpsampleStep` is used when `image` is provided (an edit, written from the images). - - `PH3ImageText2ImagePromptUpsampleStep` otherwise (text-to-image). + - `PImageOSSEditPromptUpsampleStep` is used when `image` is provided (an edit, written from the images). + - `PImageOSSText2ImagePromptUpsampleStep` otherwise (text-to-image). To render `prompt` as given, remove the step from the blocks: `blocks.sub_blocks.pop("prompt_upsampler")`. Components: @@ -278,18 +278,18 @@ class PH3ImageAutoPromptUpsampleStep(AutoPipelineBlocks): The edit prompt, replacing the instruction. """ - model_name = "p-h3-image" - block_classes = [PH3ImageEditPromptUpsampleStep, PH3ImageText2ImagePromptUpsampleStep] + model_name = "p-image-oss" + block_classes = [PImageOSSEditPromptUpsampleStep, PImageOSSText2ImagePromptUpsampleStep] block_names = ["image_conditioned", "text2image"] block_trigger_inputs = ["image", None] @property def description(self) -> str: return ( - "Prompt upsampling step: expands `prompt` into the full prompt P-H3-Image is meant to render, with the " + "Prompt upsampling step: expands `prompt` into the full prompt P-Image-OSS is meant to render, with the " "prompt upsampler (based on Qwen3.5-9B). The task follows the inputs, like the rest of the pipeline:\n" - " - `PH3ImageEditPromptUpsampleStep` is used when `image` is provided (an edit, written from the " + " - `PImageOSSEditPromptUpsampleStep` is used when `image` is provided (an edit, written from the " "images).\n" - " - `PH3ImageText2ImagePromptUpsampleStep` otherwise (text-to-image).\n" + " - `PImageOSSText2ImagePromptUpsampleStep` otherwise (text-to-image).\n" 'To render `prompt` as given, remove the step from the blocks: `blocks.sub_blocks.pop("prompt_upsampler")`.' ) diff --git a/src/diffusers/utils/dummy_pt_objects.py b/src/diffusers/utils/dummy_pt_objects.py index e11524497db0..5ba1d1e1416e 100644 --- a/src/diffusers/utils/dummy_pt_objects.py +++ b/src/diffusers/utils/dummy_pt_objects.py @@ -964,6 +964,21 @@ def from_pretrained(cls, *args, **kwargs): requires_backends(cls, ["torch"]) +class AutoencoderKLPImageOSS(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + class AutoencoderKLQwenImage(metaclass=DummyObject): _backends = ["torch"] @@ -2404,6 +2419,21 @@ def from_pretrained(cls, *args, **kwargs): requires_backends(cls, ["torch"]) +class PImageOSSTextConnector(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + class PixArtTransformer2DModel(metaclass=DummyObject): _backends = ["torch"] diff --git a/src/diffusers/utils/dummy_torch_and_transformers_objects.py b/src/diffusers/utils/dummy_torch_and_transformers_objects.py index c5f61f870a6d..d24ecfafff39 100644 --- a/src/diffusers/utils/dummy_torch_and_transformers_objects.py +++ b/src/diffusers/utils/dummy_torch_and_transformers_objects.py @@ -917,6 +917,36 @@ def from_pretrained(cls, *args, **kwargs): requires_backends(cls, ["torch", "transformers"]) +class PImageOSSBlocks(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class PImageOSSModularPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + class QwenImage21AutoBlocks(metaclass=DummyObject): _backends = ["torch", "transformers"] diff --git a/tests/models/autoencoders/test_models_autoencoder_kl_p_h3_image.py b/tests/models/autoencoders/test_models_autoencoder_kl_p_image_oss.py similarity index 81% rename from tests/models/autoencoders/test_models_autoencoder_kl_p_h3_image.py rename to tests/models/autoencoders/test_models_autoencoder_kl_p_image_oss.py index 7f857fd848df..bca5147f71fc 100644 --- a/tests/models/autoencoders/test_models_autoencoder_kl_p_h3_image.py +++ b/tests/models/autoencoders/test_models_autoencoder_kl_p_image_oss.py @@ -16,7 +16,7 @@ import pytest import torch -from diffusers import AutoencoderKLPH3Image +from diffusers import AutoencoderKLPImageOSS from diffusers.utils.torch_utils import randn_tensor from ...testing_utils import enable_full_determinism, torch_device @@ -38,10 +38,10 @@ HEIGHT = WIDTH = 32 -class AutoencoderKLPH3ImageTesterConfig(BaseModelTesterConfig): +class AutoencoderKLPImageOSSTesterConfig(BaseModelTesterConfig): @property def model_class(self): - return AutoencoderKLPH3Image + return AutoencoderKLPImageOSS @property def main_input_name(self) -> str: @@ -84,8 +84,8 @@ def get_dummy_inputs(self) -> dict: } -class TestAutoencoderKLPH3Image(AutoencoderKLPH3ImageTesterConfig, ModelTesterMixin): - """Core model tests for the P-H3-Image image autoencoder.""" +class TestAutoencoderKLPImageOSS(AutoencoderKLPImageOSSTesterConfig, ModelTesterMixin): + """Core model tests for the P-Image-OSS image autoencoder.""" @pytest.mark.skip( "`forward` runs through the `apply_forward_hook`-decorated `encode` and `decode`, and that decorator's " @@ -108,8 +108,8 @@ def test_encode_decode_single_frame(self): assert decoded.shape == sample.shape -class TestAutoencoderKLPH3ImageMemory(AutoencoderKLPH3ImageTesterConfig, MemoryTesterMixin): - """Memory optimization tests for the P-H3-Image image autoencoder.""" +class TestAutoencoderKLPImageOSSMemory(AutoencoderKLPImageOSSTesterConfig, MemoryTesterMixin): + """Memory optimization tests for the P-Image-OSS image autoencoder.""" # The encoder's reflect spatial padding has no deterministic CUDA backward # (reflection_pad3d_backward_out_cuda), so every test with a backward pass runs with determinism relaxed. @@ -117,11 +117,11 @@ def test_layerwise_casting_training(self): run_nondeterministic(super().test_layerwise_casting_training) -class TestAutoencoderKLPH3ImageTraining(AutoencoderKLPH3ImageTesterConfig, TrainingTesterMixin): - """Training tests for the P-H3-Image image autoencoder.""" +class TestAutoencoderKLPImageOSSTraining(AutoencoderKLPImageOSSTesterConfig, TrainingTesterMixin): + """Training tests for the P-Image-OSS image autoencoder.""" def test_gradient_checkpointing_is_applied(self): - super().test_gradient_checkpointing_is_applied(expected_set={"PH3ImageDownBlock3d", "PH3ImageViTDecoder"}) + super().test_gradient_checkpointing_is_applied(expected_set={"PImageOSSDownBlock3d", "PImageOSSViTDecoder"}) # The encoder's reflect spatial padding has no deterministic CUDA backward # (reflection_pad3d_backward_out_cuda), so every test with a backward pass runs with determinism relaxed. @@ -138,9 +138,9 @@ def test_gradient_checkpointing_equivalence(self): run_nondeterministic(super().test_gradient_checkpointing_equivalence) -class TestAutoencoderKLPH3ImageAttention(AutoencoderKLPH3ImageTesterConfig, AttentionTesterMixin): - """Attention processor tests for the P-H3-Image image autoencoder.""" +class TestAutoencoderKLPImageOSSAttention(AutoencoderKLPImageOSSTesterConfig, AttentionTesterMixin): + """Attention processor tests for the P-Image-OSS image autoencoder.""" -class TestAutoencoderKLPH3ImageSlicingTiling(AutoencoderKLPH3ImageTesterConfig, AutoencoderTesterMixin): - """Slicing and tiling tests for the P-H3-Image image autoencoder.""" +class TestAutoencoderKLPImageOSSSlicingTiling(AutoencoderKLPImageOSSTesterConfig, AutoencoderTesterMixin): + """Slicing and tiling tests for the P-Image-OSS image autoencoder.""" diff --git a/tests/modular_pipelines/p_h3_image/__init__.py b/tests/modular_pipelines/p_image_oss/__init__.py similarity index 100% rename from tests/modular_pipelines/p_h3_image/__init__.py rename to tests/modular_pipelines/p_image_oss/__init__.py diff --git a/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py b/tests/modular_pipelines/p_image_oss/test_modular_pipeline_p_image_oss.py similarity index 85% rename from tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py rename to tests/modular_pipelines/p_image_oss/test_modular_pipeline_p_image_oss.py index 40f998bccb20..b49a1699b79c 100644 --- a/tests/modular_pipelines/p_h3_image/test_modular_pipeline_p_h3_image.py +++ b/tests/modular_pipelines/p_image_oss/test_modular_pipeline_p_image_oss.py @@ -20,14 +20,14 @@ from PIL import Image from transformers import AutoTokenizer, Qwen3_5ForConditionalGeneration -from diffusers.modular_pipelines import PH3ImageBlocks, PH3ImageModularPipeline -from diffusers.modular_pipelines.p_h3_image.encoders import ( - PH3ImageEditTextEncoderStep, - PH3ImageEditVaeEncoderStep, - PH3ImageTextEncoderStep, +from diffusers.modular_pipelines import PImageOSSBlocks, PImageOSSModularPipeline +from diffusers.modular_pipelines.p_image_oss.encoders import ( + PImageOSSEditTextEncoderStep, + PImageOSSEditVaeEncoderStep, + PImageOSSTextEncoderStep, ) -from diffusers.modular_pipelines.p_h3_image.prompt_upsampler import ( - PH3ImageEditPromptUpsampleStep, +from diffusers.modular_pipelines.p_image_oss.prompt_upsampler import ( + PImageOSSEditPromptUpsampleStep, aspect_ratio_label, is_edit_prompt, ) @@ -43,31 +43,31 @@ ) -# The blocks each workflow of `PH3ImageBlocks` runs, in order. -P_H3_IMAGE_WORKFLOWS = { +# The blocks each workflow of `PImageOSSBlocks` runs, in order. +P_IMAGE_OSS_WORKFLOWS = { "text2image": [ - ("prompt_upsampler", "PH3ImageText2ImagePromptUpsampleStep"), - ("text_encoder", "PH3ImageTextEncoderStep"), - ("denoise.prepare_layout", "PH3ImagePrepareLayoutStep"), - ("denoise.prepare_latents", "PH3ImagePrepareLatentsStep"), - ("denoise.set_timesteps", "PH3ImageSetTimestepsStep"), - ("denoise.denoise", "PH3ImageDenoiseStep"), - ("denoise.after_denoise", "PH3ImageAfterDenoiseStep"), - ("decode", "PH3ImageDecodeStep"), + ("prompt_upsampler", "PImageOSSText2ImagePromptUpsampleStep"), + ("text_encoder", "PImageOSSTextEncoderStep"), + ("denoise.prepare_layout", "PImageOSSPrepareLayoutStep"), + ("denoise.prepare_latents", "PImageOSSPrepareLatentsStep"), + ("denoise.set_timesteps", "PImageOSSSetTimestepsStep"), + ("denoise.denoise", "PImageOSSDenoiseStep"), + ("denoise.after_denoise", "PImageOSSAfterDenoiseStep"), + ("decode", "PImageOSSDecodeStep"), ], "image_conditioned": [ - ("before_encode", "PH3ImageEditResizeStep"), - ("prompt_upsampler", "PH3ImageEditPromptUpsampleStep"), - ("text_encoder", "PH3ImageEditTextEncoderStep"), - ("vae_encoder", "PH3ImageEditVaeEncoderStep"), - ("denoise.prepare_layout", "PH3ImageEditPrepareLayoutStep"), - ("denoise.prepare_condition_latents", "PH3ImagePrepareConditionLatentsStep"), - ("denoise.prepare_latents", "PH3ImagePrepareLatentsStep"), - ("denoise.prepare_latents_image_conditioned", "PH3ImageEditPrepareLatentsStep"), - ("denoise.set_timesteps", "PH3ImageSetTimestepsStep"), - ("denoise.denoise", "PH3ImageDenoiseStep"), - ("denoise.after_denoise", "PH3ImageAfterDenoiseStep"), - ("decode", "PH3ImageDecodeStep"), + ("before_encode", "PImageOSSEditResizeStep"), + ("prompt_upsampler", "PImageOSSEditPromptUpsampleStep"), + ("text_encoder", "PImageOSSEditTextEncoderStep"), + ("vae_encoder", "PImageOSSEditVaeEncoderStep"), + ("denoise.prepare_layout", "PImageOSSEditPrepareLayoutStep"), + ("denoise.prepare_condition_latents", "PImageOSSPrepareConditionLatentsStep"), + ("denoise.prepare_latents", "PImageOSSPrepareLatentsStep"), + ("denoise.prepare_latents_image_conditioned", "PImageOSSEditPrepareLatentsStep"), + ("denoise.set_timesteps", "PImageOSSSetTimestepsStep"), + ("denoise.denoise", "PImageOSSDenoiseStep"), + ("denoise.after_denoise", "PImageOSSAfterDenoiseStep"), + ("decode", "PImageOSSDecodeStep"), ], } @@ -75,13 +75,13 @@ "prompt_upsampler": "Qwen3_5ForConditionalGeneration", "prompt_upsampler_processor": "AutoProcessor", "text_encoder": "Qwen3VLForConditionalGeneration", - "text_connector": "PH3ImageTextConnector", + "text_connector": "PImageOSSTextConnector", "tokenizer": "Qwen2Tokenizer", "processor": "Qwen3VLProcessor", "scheduler": "MiniMaxH3Scheduler", "audio_scheduler": "MiniMaxH3Scheduler", "transformer": "MiniMaxH3Transformer3DModel", - "vae": "AutoencoderKLPH3Image", + "vae": "AutoencoderKLPImageOSS", "image_processor": "VaeImageProcessor", } _SAMPLE_SIGMAS = [1.0, 0.984753, 0.967905, 0.951688, 0.925372, 0.881923, 0.810083, 0.528718, 0.0] @@ -92,7 +92,7 @@ "attention_kwargs": None, "output_type": "pil", } -P_H3_IMAGE_WORKFLOW_DEFAULTS = { +P_IMAGE_OSS_WORKFLOW_DEFAULTS = { "text2image": { "components": _SHARED_COMPONENTS, "configs": {"sample_sigmas": _SAMPLE_SIGMAS}, @@ -148,11 +148,11 @@ def generate(self, input_ids, **kwargs): yield -class PH3ImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): - """The `text2image` and `image_conditioned` requests of [`PH3ImageBlocks`].""" +class PImageOSSModularPipelineTesterConfig(BaseModularPipelineTesterConfig): + """The `text2image` and `image_conditioned` requests of [`PImageOSSBlocks`].""" - pipeline_class = PH3ImageModularPipeline - pipeline_blocks_class = PH3ImageBlocks + pipeline_class = PImageOSSModularPipeline + pipeline_blocks_class = PImageOSSBlocks pretrained_model_name_or_path = "hf-internal-testing/tiny-p-h3-image-modular-pipe" # One 4-channel latent frame for a 32 x 32 image. expected_latents_shape = (1, 4, 1, 2, 2) @@ -163,8 +163,8 @@ class PH3ImageModularPipelineTesterConfig(BaseModularPipelineTesterConfig): # `num_inference_steps`. optional_params = frozenset(["latents", "output_type"]) output_name = "images" - expected_workflow_blocks = P_H3_IMAGE_WORKFLOWS - expected_workflow_defaults = P_H3_IMAGE_WORKFLOW_DEFAULTS + expected_workflow_blocks = P_IMAGE_OSS_WORKFLOWS + expected_workflow_defaults = P_IMAGE_OSS_WORKFLOW_DEFAULTS def get_dummy_inputs(self, seed=0): return { @@ -187,12 +187,12 @@ def get_edit_inputs(self, seed=0): } -class TestPH3ImageModularPipelineFast(PH3ImageModularPipelineTesterConfig, ModularPipelineTesterMixin): - @pytest.mark.skip(reason="P-H3-Image packs one request into one sequence, so a batch of prompts is not a thing.") +class TestPImageOSSModularPipelineFast(PImageOSSModularPipelineTesterConfig, ModularPipelineTesterMixin): + @pytest.mark.skip(reason="P-Image-OSS packs one request into one sequence, so a batch of prompts is not a thing.") def test_inference_batch_consistent(self): pass - @pytest.mark.skip(reason="P-H3-Image packs one request into one sequence, so a batch of prompts is not a thing.") + @pytest.mark.skip(reason="P-Image-OSS packs one request into one sequence, so a batch of prompts is not a thing.") def test_inference_batch_single_identical(self): pass @@ -317,7 +317,7 @@ def test_custom_sigmas(self): @pytest.mark.parametrize("edit", [False, True], ids=["text2image", "image_conditioned"]) def test_text_encoder_block_standalone(self, edit): r"""Both text encoder blocks run on their own. Vision tokens of an edit are tagged as video (`0`).""" - block = PH3ImageEditTextEncoderStep() if edit else PH3ImageTextEncoderStep() + block = PImageOSSEditTextEncoderStep() if edit else PImageOSSTextEncoderStep() pipe = block.init_pipeline(self.pretrained_model_name_or_path) pipe.load_components(dtype=torch.float32) @@ -331,7 +331,7 @@ def test_text_encoder_block_standalone(self, edit): def test_vae_encoder_block_standalone(self): r"""The edit's VAE encoder block runs on its own.""" - pipe = PH3ImageEditVaeEncoderStep().init_pipeline(self.pretrained_model_name_or_path) + pipe = PImageOSSEditVaeEncoderStep().init_pipeline(self.pretrained_model_name_or_path) pipe.load_components(dtype=torch.float32) condition_latents = pipe(reference_images=[Image.new("RGB", (64, 32))], output="condition_latents") @@ -371,7 +371,7 @@ def test_size_rounded_to_multiples_of_32(self, edit): inputs = self.get_edit_inputs() if edit else self.get_dummy_inputs() inputs["height"], inputs["width"] = 30, 70 - logger = logging.get_logger("diffusers.modular_pipelines.p_h3_image.before_encoder") + logger = logging.get_logger("diffusers.modular_pipelines.p_image_oss.before_encoder") with CaptureLogger(logger) as cap: images = pipe(**inputs, output="images") @@ -444,7 +444,7 @@ def test_prompt_upsampler_rejects_an_unusable_prompt(self): def test_prompt_upsampler_image_limit(self): r"""The upsampler block run on its own takes the same 8 images as the pipeline.""" - pipe = PH3ImageEditPromptUpsampleStep().init_pipeline(self.pretrained_model_name_or_path) + pipe = PImageOSSEditPromptUpsampleStep().init_pipeline(self.pretrained_model_name_or_path) pipe.load_components(dtype=torch.float32) with pytest.raises(ValueError, match="at most 8 images"): @@ -468,7 +468,7 @@ def test_upsampler_helpers(self): def test_edit_prefix(self): r"""A short instruction gets the `"Edit . "` prefix; a prompt that already has it is encoded as given.""" - block = PH3ImageEditTextEncoderStep() + block = PImageOSSEditTextEncoderStep() pipe = block.init_pipeline(self.pretrained_model_name_or_path) pipe.load_components(dtype=torch.float32) vision_images = [Image.new("RGB", (32, 32))] @@ -488,13 +488,13 @@ def test_edit_too_many_images(self): pipe(**inputs) -class TestPH3ImageModularPipelineLoading(PH3ImageModularPipelineTesterConfig, ModularLoadingTesterMixin): +class TestPImageOSSModularPipelineLoading(PImageOSSModularPipelineTesterConfig, ModularLoadingTesterMixin): pass -class TestPH3ImageModularPipelineWorkflow(PH3ImageModularPipelineTesterConfig, ModularWorkflowTesterMixin): +class TestPImageOSSModularPipelineWorkflow(PImageOSSModularPipelineTesterConfig, ModularWorkflowTesterMixin): pass -class TestPH3ImageModularPipelineMemory(PH3ImageModularPipelineTesterConfig, ModularMemoryTesterMixin): +class TestPImageOSSModularPipelineMemory(PImageOSSModularPipelineTesterConfig, ModularMemoryTesterMixin): pass From 3c7bda1b1d52fdeee306437f09e6a8e86aacfeeb Mon Sep 17 00:00:00 2001 From: Sara Han <127759186+sdiazlor@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:33:09 +0100 Subject: [PATCH 2/2] fix: small clarification in doc about the aliases --- docs/source/en/api/pipelines/p_image_oss.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/api/pipelines/p_image_oss.md b/docs/source/en/api/pipelines/p_image_oss.md index 496683a539ae..4d2133c06e05 100644 --- a/docs/source/en/api/pipelines/p_image_oss.md +++ b/docs/source/en/api/pipelines/p_image_oss.md @@ -14,7 +14,7 @@ specific language governing permissions and limitations under the License. --> P-Image-OSS is an image model by Pruna AI based on [MiniMax-H3](./minimax_h3), for text-to-image generation and instruction-based image editing. You can find the checkpoint at [PrunaAI/P-Image-OSS](https://huggingface.co/PrunaAI/P-Image-OSS). > [!TIP] -> P-Image-OSS was previously released as P-H3-Image. The former class names (`PH3ImageModularPipeline`, `PH3ImageBlocks`, `PH3ImageTextConnector`, and `AutoencoderKLPH3Image`) are kept as deprecated aliases of the new ones. +> P-Image-OSS was previously released as P-H3-Image. The former class names (`PH3ImageModularPipeline`, `PH3ImageBlocks`, `PH3ImageTextConnector`, and `AutoencoderKLPH3Image`) are kept as aliases of the new ones. ```py import torch