Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 31 additions & 7 deletions src/diffusers/hooks/context_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import torch.distributed._functional_collectives as funcol

from ..models._modeling_parallel import (
Conditional,
ContextParallelConfig,
ContextParallelInput,
ContextParallelModelPlan,
Expand Down Expand Up @@ -85,10 +86,17 @@ def apply_context_parallel(
"""Apply context parallel on a model."""
logger.debug(f"Applying context parallel with CP mesh: {parallel_config._mesh} and plan: {plan}")

unmatched = []
for module_id, cp_model_plan in plan.items():
submodule = _get_submodule_by_name(module, module_id)
conditional = isinstance(cp_model_plan, Conditional)
if conditional:
cp_model_plan = cp_model_plan.value
submodule = _get_submodule_by_name(module, module_id, conditional)
if not isinstance(submodule, list):
submodule = [submodule]
if conditional and len(submodule) == 0:
unmatched.append(module_id)
continue

logger.debug(f"Applying ContextParallelHook to {module_id=} identifying a total of {len(submodule)} modules")

Expand All @@ -108,10 +116,19 @@ def apply_context_parallel(
registry = HookRegistry.check_if_exists_or_initialize(m)
registry.register_hook(hook, hook_name)

if unmatched:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i don't think we need warn here, it's expected behavior -> change to debug maybe?

logger.debug(
f"Skipping the `Conditional` context parallel plan entries that match no module of "
f"'{module.__class__.__name__}': {unmatched}."
)


def remove_context_parallel(module: torch.nn.Module, plan: dict[str, ContextParallelModelPlan]) -> None:
for module_id, cp_model_plan in plan.items():
submodule = _get_submodule_by_name(module, module_id)
conditional = isinstance(cp_model_plan, Conditional)
if conditional:
cp_model_plan = cp_model_plan.value
submodule = _get_submodule_by_name(module, module_id, conditional)
if not isinstance(submodule, list):
submodule = [submodule]

Expand Down Expand Up @@ -354,13 +371,17 @@ def _all_gather_anything(tensor: torch.Tensor, dim: int, group: dist.device_mesh
return gathered_tensor


def _get_submodule_by_name(model: torch.nn.Module, name: str) -> torch.nn.Module | list[torch.nn.Module]:
def _get_submodule_by_name(
model: torch.nn.Module, name: str, conditional: bool = False
) -> torch.nn.Module | list[torch.nn.Module]:
if name.count("*") > 1:
raise ValueError("Wildcard '*' can only be used once in the name")
return _find_submodule_by_name(model, name)
return _find_submodule_by_name(model, name, conditional)


def _find_submodule_by_name(model: torch.nn.Module, name: str) -> torch.nn.Module | list[torch.nn.Module]:
def _find_submodule_by_name(
model: torch.nn.Module, name: str, conditional: bool = False
) -> torch.nn.Module | list[torch.nn.Module]:
if name == "":
return model
first_atom, remaining_name = name.split(".", 1) if "." in name else (name, "")
Expand All @@ -369,14 +390,17 @@ def _find_submodule_by_name(model: torch.nn.Module, name: str) -> torch.nn.Modul
raise ValueError("Wildcard '*' can only be used with ModuleList")
submodules = []
for submodule in model:
subsubmodules = _find_submodule_by_name(submodule, remaining_name)
subsubmodules = _find_submodule_by_name(submodule, remaining_name, conditional)
if not isinstance(subsubmodules, list):
subsubmodules = [subsubmodules]
submodules.extend(subsubmodules)
return submodules
else:
if conditional and getattr(model, first_atom, None) is None:
# A `Conditional` entry matches nothing where its module is missing or set to `None`.
return []
if hasattr(model, first_atom):
submodule = getattr(model, first_atom)
return _find_submodule_by_name(submodule, remaining_name)
return _find_submodule_by_name(submodule, remaining_name, conditional)
else:
raise ValueError(f"'{first_atom}' is not a submodule of '{model.__class__.__name__}'")
55 changes: 46 additions & 9 deletions src/diffusers/hooks/tensor_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import torch

from ..models._modeling_parallel import TensorParallelConfig
from ..models._modeling_parallel import Conditional, TensorParallelConfig
from ..utils import get_logger, is_peft_available


Expand Down Expand Up @@ -183,45 +183,82 @@ def resolve_tp_shard_specs(model: torch.nn.Module, tp_plan: dict, tp_degree: int
return specs


def _resolve_tp_plan(model: torch.nn.Module, tp_plan: dict) -> list:
def _maybe_get_submodule(module: torch.nn.Module, path: str) -> "torch.nn.Module | None":
"""Resolve the dotted `path` on `module`, or return `None` if an attribute along it is missing or `None`."""
for atom in path.split("."):
module = getattr(module, atom, None)
if module is None:
return None
return module


def _resolve_tp_plan(model: torch.nn.Module, tp_plan: dict, log_unmatched: bool = False) -> list:
"""Group a flat `_tp_plan` into per-block `(submodule, {relative_path: style})` plans.

Each glob is split at its single `*`; the prefix must resolve to a `ModuleList` and the suffix is the per-element
key. Grouping by block lets the caller issue one `parallelize_module` call per block, which `RowwiseParallel` needs
to attach its input redistribution at the block boundary.

A `Conditional` entry is unwrapped here and kept only for the blocks where its module exists, so the returned plans
hold plain styles. `log_unmatched` logs the `Conditional` entries that match no module; it is off by default
because the plan is resolved several times per model.

Example: when `transformer_blocks` is a `ModuleList` of length 2, the input `{"transformer_blocks.*.ff.linear_out":
"rowwise"}` returns `[(transformer_blocks[0], {"ff.linear_out": "rowwise"}), (transformer_blocks[1],
{"ff.linear_out": "rowwise"})]`.
"""
grouped: dict[int, tuple] = {}
order: list[int] = []
unmatched: list[str] = []

for pattern, style in tp_plan.items():
if pattern.count("*") > 1:
raise ValueError(f"Wildcard '*' can only be used once in a `_tp_plan` key, got '{pattern}'.")

conditional = isinstance(style, Conditional)
if conditional:
style = style.value

if "*" in pattern:
prefix, _, suffix = pattern.partition("*")
container = model
for atom in prefix.strip(".").split("."):
container = getattr(container, atom)
if not isinstance(container, torch.nn.ModuleList):
prefix = prefix.strip(".")
if conditional:
container = _maybe_get_submodule(model, prefix)
else:
container = model
for atom in prefix.split("."):
container = getattr(container, atom)
if conditional and container is None:
blocks = []
elif not isinstance(container, torch.nn.ModuleList):
raise ValueError(
f"`_tp_plan` wildcard '{pattern}' must expand over a `ModuleList`, but "
f"'{prefix.strip('.')}' resolved to '{container.__class__.__name__}'."
f"'{prefix}' resolved to '{container.__class__.__name__}'."
)
relative, blocks = suffix.strip("."), list(container)
else:
blocks = list(container)
relative = suffix.strip(".")
else:
relative, blocks = pattern, [model]

if conditional:
blocks = [block for block in blocks if _maybe_get_submodule(block, relative) is not None]
if not blocks:
unmatched.append(pattern)

for block in blocks:
key = id(block)
if key not in grouped:
grouped[key] = (block, {})
order.append(key)
grouped[key][1][relative] = style

if log_unmatched and unmatched:
logger.debug(
f"Skipping the `Conditional` `_tp_plan` entries that match no module of '{model.__class__.__name__}': "
f"{unmatched}."
)

return [grouped[key] for key in order]


Expand Down Expand Up @@ -512,7 +549,7 @@ def apply_tensor_parallel(
)

backend = "neuron" if tp_mesh.device_type == "neuron" else "default"
groups = _resolve_tp_plan(model, tp_plan)
groups = _resolve_tp_plan(model, tp_plan, log_unmatched=True)
logger.debug(f"Applying tensor parallel (backend={backend}) over {len(groups)} module group(s) on mesh {tp_mesh}.")

from torch.distributed.tensor.parallel import parallelize_module
Expand Down
23 changes: 21 additions & 2 deletions src/diffusers/models/_modeling_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# limitations under the License.

from dataclasses import dataclass
from typing import TYPE_CHECKING, Literal
from typing import TYPE_CHECKING, Any, Literal

import torch
import torch.distributed as dist
Expand Down Expand Up @@ -285,6 +285,25 @@ def __repr__(self):
return f"ContextParallelOutput(gather_dim={self.gather_dim}, expected_dims={self.expected_dims})"


@dataclass(frozen=True)
class Conditional:
"""
Marks a `_cp_plan` or `_tp_plan` entry whose module does not exist on every instance of the model class. For
example, different checkpoints of the same model class might have different layers / modules.

A plan entry must resolve to a module, so a typo in a key raises. Wrap the value of an entry in `Conditional` when
its module depends on the config or the checkpoint. The entry is skipped wherever the module is missing or set to
`None`, and a debug message is logged if it matches no module at all.

Args:
value:
The value the entry would have without the wrapper, e.g. `Conditional("colwise")` in a `_tp_plan` or
`Conditional(ContextParallelOutput(gather_dim=1, expected_dims=3))` in a `_cp_plan`.
"""

value: Any


# A dictionary where keys denote the input to be split across context parallel region, and the
# value denotes the sharding configuration.
# If the key is a string, it denotes the name of the parameter in the forward function.
Expand All @@ -300,7 +319,7 @@ def __repr__(self):

# A dictionary where keys denote the module id, and the value denotes how the inputs/outputs of
# the module should be split/gathered across context parallel region.
ContextParallelModelPlan = dict[str, ContextParallelInputType | ContextParallelOutputType]
ContextParallelModelPlan = dict[str, ContextParallelInputType | ContextParallelOutputType | Conditional]


# Example of a ContextParallelModelPlan (QwenImageTransformer2DModel):
Expand Down
2 changes: 1 addition & 1 deletion src/diffusers/models/modeling_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1220,7 +1220,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike | None
tp_config = (
parallel_config
if isinstance(parallel_config, TensorParallelConfig)
else parallel_config.tensor_parallel_config
else getattr(parallel_config, "tensor_parallel_config", None)

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To support passing bare ContextParallelConfig to from_pretrained() just like TensorParallelConfig.

)
if tp_config is not None and tp_config.tp_degree == 1 and tp_config.mesh is None:
# Nothing to shard, so take the ordinary loader rather than building 1-rank DTensors.
Expand Down
147 changes: 147 additions & 0 deletions tests/models/test_parallelism_conditional_layers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
# coding=utf-8
# Copyright 2026 HuggingFace Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp

from diffusers.models._modeling_parallel import ContextParallelConfig, TensorParallelConfig
from diffusers.utils import logging

from ..testing_utils import (
CaptureLogger,
is_context_parallel,
is_tensor_parallel,
require_torch_multi_accelerator,
torch_device,
)
from .testing_utils.parallelism import DEVICE_CONFIG, DummyConditionalPlanModel, _find_free_port


def _conditional_plan_worker(rank, world_size, master_port, checkpoint_dir, parallelism, inputs_dict, return_dict):
"""Load `DummyConditionalPlanModel` with a `parallel_config` and run a forward pass.

Rank 0 reports its output, what the hooks logged while loading, and where the plan was applied: the parameters
sharded into `DTensor`s for tensor parallelism, or the modules carrying a hook for context parallelism.
"""
try:
os.environ["MASTER_ADDR"] = "localhost"
os.environ["MASTER_PORT"] = str(master_port)
os.environ["RANK"] = str(rank)
os.environ["WORLD_SIZE"] = str(world_size)

device_config = DEVICE_CONFIG.get(torch_device, DEVICE_CONFIG["cuda"])
dist.init_process_group(backend=device_config["backend"], rank=rank, world_size=world_size)
device_config["module"].set_device(rank)
device = torch.device(f"{torch_device}:{rank}")

from torch.distributed.tensor import DTensor

if parallelism == "tensor":
parallel_config = TensorParallelConfig(tp_degree=world_size)
else:
parallel_config = ContextParallelConfig(ulysses_degree=world_size)

# A skipped `Conditional` entry is logged at debug level.
hooks_logger = logging.get_logger("diffusers.hooks")
hooks_logger.setLevel(logging.DEBUG)
with CaptureLogger(hooks_logger) as cap_logger:
model = DummyConditionalPlanModel.from_pretrained(checkpoint_dir, parallel_config=parallel_config)
if parallelism == "context":
# A tensor-parallel load already places each rank's shard on its device.
model.to(device)

inputs_on_device = {k: v.to(device) for k, v in inputs_dict.items()}
with torch.no_grad():
output = model(**inputs_on_device)

if rank == 0:
if parallelism == "tensor":
applied = [name for name, param in model.state_dict().items() if isinstance(param, DTensor)]
else:
applied = [name for name, module in model.named_modules() if hasattr(module, "_diffusers_hook")]

return_dict["status"] = "success"
return_dict["output"] = output.cpu().tolist()
return_dict["logs"] = cap_logger.out
return_dict["applied"] = applied

except Exception as e:
if rank == 0:
return_dict["status"] = "error"
return_dict["error"] = f"{type(e).__name__}: {e}"
finally:
if dist.is_initialized():
dist.destroy_process_group()


@require_torch_multi_accelerator
class TestConditionalPlanEntries:
"""`Conditional` entries of a `_tp_plan` / `_cp_plan`, on a checkpoint with and without the conditional layers."""

unmatched_message = "match no module of 'DummyConditionalPlanModel'"

def _save_and_load_with_parallelism(self, tmp_path, parallelism, use_conditional_layers):
"""Save the model, load it on two ranks with parallelism, and check the output against a single device."""
if not torch.distributed.is_available():
pytest.skip("torch.distributed is not available.")

torch.manual_seed(0)
model = DummyConditionalPlanModel(use_conditional_layers=use_conditional_layers).eval()
inputs_dict = {"hidden_states": torch.randn(1, 8, 8), "position_ids": torch.arange(8)}
with torch.no_grad():
reference = model(**inputs_dict)
model.save_pretrained(tmp_path)

world_size = 2
return_dict = mp.Manager().dict()
mp.spawn(
_conditional_plan_worker,
args=(world_size, _find_free_port(), str(tmp_path), parallelism, inputs_dict, return_dict),
nprocs=world_size,
join=True,
)

assert return_dict.get("status") == "success", (
f"Loading with {parallelism} parallelism failed: {return_dict.get('error', 'Unknown error')}"
)
torch.testing.assert_close(reference, torch.tensor(return_dict["output"]), atol=1e-4, rtol=1e-4)
return return_dict

@is_tensor_parallel
@pytest.mark.parametrize("use_conditional_layers", [True, False])
def test_tensor_parallel(self, tmp_path, use_conditional_layers):
result = self._save_and_load_with_parallelism(tmp_path, "tensor", use_conditional_layers)

if use_conditional_layers:
gate_params = {f"blocks.{i}.gate_proj.{name}" for i in range(2) for name in ("weight", "bias")}
assert gate_params <= set(result["applied"])
assert self.unmatched_message not in result["logs"]
else:
assert f"{self.unmatched_message}: ['blocks.*.gate_proj']" in result["logs"]

@is_context_parallel
@pytest.mark.parametrize("use_conditional_layers", [True, False])
def test_context_parallel(self, tmp_path, use_conditional_layers):
result = self._save_and_load_with_parallelism(tmp_path, "context", use_conditional_layers)

if use_conditional_layers:
assert "pos_embed" in result["applied"]
assert self.unmatched_message not in result["logs"]
else:
assert f"{self.unmatched_message}: ['pos_embed']" in result["logs"]
Loading
Loading