From e88275ff618e02980f1b52d6ab90bca0c57d89f1 Mon Sep 17 00:00:00 2001 From: JingyaHuang Date: Wed, 28 Oct 2026 11:21:57 +0000 Subject: [PATCH 01/11] fix: Fix default RNG synchronization for ditribution --- src/diffusers/models/modeling_utils.py | 5 ++++- src/diffusers/utils/distributed_utils.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/diffusers/models/modeling_utils.py b/src/diffusers/models/modeling_utils.py index 469906613a5e..471ae19f2d0d 100644 --- a/src/diffusers/models/modeling_utils.py +++ b/src/diffusers/models/modeling_utils.py @@ -61,7 +61,7 @@ is_torch_version, logging, ) -from ..utils.distributed_utils import is_torch_dist_rank_zero +from ..utils.distributed_utils import is_torch_dist_rank_zero, sync_default_rng_seed from ..utils.hub_utils import PushToHubMixin, load_or_create_model_card, populate_model_card from ..utils.torch_utils import empty_device_cache from ._modeling_parallel import ( @@ -1737,6 +1737,9 @@ def _resolve_parallel_config( # `config.setup()` records the mesh resolved above onto the config; see `ParallelConfig.setup`. config.setup(rank, world_size, device, mesh=mesh) self._parallel_config = config + + seed_mesh = config.context_parallel_config._flattened_mesh if config.context_parallel_config else mesh + sync_default_rng_seed(group=seed_mesh.get_group()) return config def enable_parallelism( diff --git a/src/diffusers/utils/distributed_utils.py b/src/diffusers/utils/distributed_utils.py index 239b7b26200d..7115c96cddee 100644 --- a/src/diffusers/utils/distributed_utils.py +++ b/src/diffusers/utils/distributed_utils.py @@ -34,3 +34,16 @@ def is_torch_dist_rank_zero() -> bool: return dist_module.get_rank() == 0 except (RuntimeError, ValueError): return True + + +def sync_default_rng_seed(group=None) -> None: + """Reset the default RNGs of every rank in `group` to the seed of the group's first rank. + + Under tensor or context parallelism, every rank must draw the same random numbers (initial latents, scheduler + noise), or the sharded activations stop matching and the output is noise. Fresh processes start from different + seeds, so without this, a pipeline call without a `generator` breaks. Ranks are reset even when their seeds already + match, because the same seed does not mean the same position in the stream. + """ + seeds = [None] * torch.distributed.get_world_size(group) + torch.distributed.all_gather_object(seeds, torch.initial_seed(), group=group) + torch.manual_seed(seeds[0]) From dbd5332c3ef036c49ca07103fa6347da4fd3c1c3 Mon Sep 17 00:00:00 2001 From: JingyaHuang Date: Wed, 28 Oct 2026 13:23:14 +0000 Subject: [PATCH 02/11] fix: use broadcast for just one rank instead of all gather all ranks --- src/diffusers/models/modeling_utils.py | 4 ++-- src/diffusers/utils/distributed_utils.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/diffusers/models/modeling_utils.py b/src/diffusers/models/modeling_utils.py index 471ae19f2d0d..95ae6b21263c 100644 --- a/src/diffusers/models/modeling_utils.py +++ b/src/diffusers/models/modeling_utils.py @@ -1738,8 +1738,8 @@ def _resolve_parallel_config( config.setup(rank, world_size, device, mesh=mesh) self._parallel_config = config - seed_mesh = config.context_parallel_config._flattened_mesh if config.context_parallel_config else mesh - sync_default_rng_seed(group=seed_mesh.get_group()) + if mesh is not None: + sync_default_rng_seed(device=config._device) return config def enable_parallelism( diff --git a/src/diffusers/utils/distributed_utils.py b/src/diffusers/utils/distributed_utils.py index 7115c96cddee..0723d297f088 100644 --- a/src/diffusers/utils/distributed_utils.py +++ b/src/diffusers/utils/distributed_utils.py @@ -36,7 +36,7 @@ def is_torch_dist_rank_zero() -> bool: return True -def sync_default_rng_seed(group=None) -> None: +def sync_default_rng_seed(group=None, device=None) -> None: """Reset the default RNGs of every rank in `group` to the seed of the group's first rank. Under tensor or context parallelism, every rank must draw the same random numbers (initial latents, scheduler @@ -44,6 +44,7 @@ def sync_default_rng_seed(group=None) -> None: seeds, so without this, a pipeline call without a `generator` breaks. Ranks are reset even when their seeds already match, because the same seed does not mean the same position in the stream. """ - seeds = [None] * torch.distributed.get_world_size(group) - torch.distributed.all_gather_object(seeds, torch.initial_seed(), group=group) + seeds = [torch.initial_seed()] + src = torch.distributed.get_global_rank(group, 0) if group is not None else 0 + torch.distributed.broadcast_object_list(seeds, src=src, group=group, device=device) torch.manual_seed(seeds[0]) From f6ad3c8a89ddeba1800ee1ff18dfee7098b894e0 Mon Sep 17 00:00:00 2001 From: JingyaHuang Date: Wed, 28 Oct 2026 13:25:23 +0000 Subject: [PATCH 03/11] removal: simplify the docstring --- src/diffusers/utils/distributed_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/diffusers/utils/distributed_utils.py b/src/diffusers/utils/distributed_utils.py index 0723d297f088..8bd4ac6310cb 100644 --- a/src/diffusers/utils/distributed_utils.py +++ b/src/diffusers/utils/distributed_utils.py @@ -40,9 +40,8 @@ def sync_default_rng_seed(group=None, device=None) -> None: """Reset the default RNGs of every rank in `group` to the seed of the group's first rank. Under tensor or context parallelism, every rank must draw the same random numbers (initial latents, scheduler - noise), or the sharded activations stop matching and the output is noise. Fresh processes start from different - seeds, so without this, a pipeline call without a `generator` breaks. Ranks are reset even when their seeds already - match, because the same seed does not mean the same position in the stream. + noise), or the sharded activations stop matching and the output is noise. Ranks are reset even when their seeds + already match, because the same seed does not mean the same position in the stream. """ seeds = [torch.initial_seed()] src = torch.distributed.get_global_rank(group, 0) if group is not None else 0 From 27d63cdde14443ea378a43bc2c5caacdc919b39e Mon Sep 17 00:00:00 2001 From: JingyaHuang Date: Wed, 28 Oct 2026 13:27:15 +0000 Subject: [PATCH 04/11] test: seeds matching test --- tests/others/test_distributed_utils.py | 142 +++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 tests/others/test_distributed_utils.py diff --git a/tests/others/test_distributed_utils.py b/tests/others/test_distributed_utils.py new file mode 100644 index 000000000000..0f84607c1838 --- /dev/null +++ b/tests/others/test_distributed_utils.py @@ -0,0 +1,142 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import socket +from unittest.mock import MagicMock, Mock, patch + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from diffusers import ModelMixin +from diffusers.models._modeling_parallel import ContextParallelConfig, ParallelConfig, TensorParallelConfig +from diffusers.utils.distributed_utils import sync_default_rng_seed + + +def _find_free_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def _sync_seed_worker(rank, world_size, master_port, seeds, advances, return_dict): + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + dist.init_process_group(backend="gloo", rank=rank, world_size=world_size) + try: + torch.manual_seed(seeds[rank]) + torch.randn(advances[rank]) + sync_default_rng_seed(device=torch.device("cpu")) + assert torch.initial_seed() == seeds[0] + return_dict[rank] = torch.randn(4) + finally: + dist.destroy_process_group() + + +def _run(seeds, advances): + world_size = len(seeds) + return_dict = mp.Manager().dict() + mp.spawn( + _sync_seed_worker, + args=(world_size, _find_free_port(), seeds, advances, return_dict), + nprocs=world_size, + join=True, + ) + return [return_dict[rank] for rank in range(world_size)] + + +def _expected_draws(seed): + torch.manual_seed(seed) + return torch.randn(4) + + +def test_sync_default_rng_seed_aligns_different_seeds(): + draws = _run(seeds=[0, 1], advances=[1, 2]) + assert torch.equal(draws[0], _expected_draws(0)) + assert torch.equal(draws[1], _expected_draws(0)) + + +def test_sync_default_rng_seed_aligns_shared_seed_at_different_offsets(): + # Same seed, but rank 1 has drawn more random numbers, so the streams are out of step until they are reset. + draws = _run(seeds=[7, 7], advances=[1, 2]) + assert torch.equal(draws[0], _expected_draws(7)) + assert torch.equal(draws[1], _expected_draws(7)) + + +@pytest.mark.parametrize("seed", [2**63, 2**64 - 1]) +def test_sync_default_rng_seed_preserves_unsigned_seed(seed): + draws = _run(seeds=[seed, 1], advances=[1, 2]) + for draw in draws: + assert torch.equal(draw, _expected_draws(seed)) + + +def test_sync_default_rng_seed_uses_explicit_device(): + device = torch.device("cuda:1") + with ( + patch("torch.initial_seed", return_value=2**64 - 1), + patch("torch.distributed.broadcast_object_list") as broadcast, + patch("torch.manual_seed") as manual_seed, + ): + sync_default_rng_seed(device=device) + + broadcast.assert_called_once_with([2**64 - 1], src=0, group=None, device=device) + manual_seed.assert_called_once_with(2**64 - 1) + + +def test_sync_default_rng_seed_uses_subgroup_source(): + group = Mock() + device = torch.device("cpu") + with ( + patch("torch.initial_seed", return_value=7), + patch("torch.distributed.get_global_rank", return_value=2) as get_global_rank, + patch("torch.distributed.broadcast_object_list") as broadcast, + patch("torch.manual_seed"), + ): + sync_default_rng_seed(group=group, device=device) + + get_global_rank.assert_called_once_with(group, 0) + broadcast.assert_called_once_with([7], src=2, group=group, device=device) + + +@pytest.mark.parametrize("parallel_type", ["tp", "cp", "empty"]) +def test_resolve_parallel_config_syncs_world_only_for_parallel_models(parallel_type): + mesh = MagicMock() + mesh.size.return_value = 2 + mesh.ndim = 2 + mesh.mesh_dim_names = ("ring", "ulysses") + if parallel_type == "tp": + config = TensorParallelConfig(mesh=mesh) + elif parallel_type == "cp": + config = ContextParallelConfig(ring_degree=2, mesh=mesh) + else: + config = ParallelConfig() + + with ( + patch("torch.distributed.is_available", return_value=True), + patch("torch.distributed.is_initialized", return_value=True), + patch("torch.distributed.get_rank", return_value=1), + patch("torch.distributed.get_world_size", return_value=4), + patch("torch._C._get_accelerator", return_value=torch.device("cuda")), + patch("torch.accelerator.device_count", return_value=4), + patch("diffusers.models.modeling_utils.sync_default_rng_seed") as sync_seed, + ): + resolved = ModelMixin()._resolve_parallel_config(config) + + if parallel_type == "empty": + sync_seed.assert_not_called() + else: + sync_seed.assert_called_once_with(device=torch.device("cuda:1")) + assert resolved._device == torch.device("cuda:1") From a94cb2801edd0a976a3874344eb883720ff154a4 Mon Sep 17 00:00:00 2001 From: JingyaHuang Date: Wed, 28 Oct 2026 13:43:07 +0000 Subject: [PATCH 05/11] test: improve, get into existed tests --- tests/models/testing_utils/parallelism.py | 17 +++ tests/others/test_distributed_utils.py | 142 ---------------------- 2 files changed, 17 insertions(+), 142 deletions(-) delete mode 100644 tests/others/test_distributed_utils.py diff --git a/tests/models/testing_utils/parallelism.py b/tests/models/testing_utils/parallelism.py index de19a7edd212..98a6f917380e 100644 --- a/tests/models/testing_utils/parallelism.py +++ b/tests/models/testing_utils/parallelism.py @@ -24,6 +24,7 @@ from diffusers.models._modeling_parallel import ContextParallelConfig, TensorParallelConfig from diffusers.models.attention_dispatch import AttentionBackendName, _AttentionBackendRegistry from diffusers.utils import SAFE_WEIGHTS_INDEX_NAME +from diffusers.utils.torch_utils import randn_tensor from ...testing_utils import ( is_attention, @@ -108,8 +109,16 @@ def _context_parallel_worker( # Enable context parallelism cp_config = ContextParallelConfig(**cp_dict) + randn_tensor((rank + 1,), device=device) model.enable_parallelism(config=cp_config) + latents = randn_tensor((1, 4, 8, 8), device=device) + gathered_latents = [torch.empty_like(latents) for _ in range(world_size)] + dist.all_gather(gathered_latents, latents) + assert all(torch.equal(latents, other) for other in gathered_latents), ( + "Unseeded latents differ across context-parallel ranks." + ) + # Run forward pass with torch.no_grad(): output = model(**inputs_on_device, return_dict=False)[0] @@ -292,8 +301,16 @@ def _tensor_parallel_worker( inputs_on_device = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()} # Shard the model across all ranks; the device mesh is built from `tp_degree` on the active accelerator. + randn_tensor((rank + 1,), device=device) model.enable_parallelism(config=TensorParallelConfig(tp_degree=world_size)) + latents = randn_tensor((1, 4, 8, 8), device=device) + gathered_latents = [torch.empty_like(latents) for _ in range(world_size)] + dist.all_gather(gathered_latents, latents) + assert all(torch.equal(latents, other) for other in gathered_latents), ( + "Unseeded latents differ across tensor-parallel ranks." + ) + with torch.no_grad(): output = model(**inputs_on_device, return_dict=False)[0] diff --git a/tests/others/test_distributed_utils.py b/tests/others/test_distributed_utils.py deleted file mode 100644 index 0f84607c1838..000000000000 --- a/tests/others/test_distributed_utils.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright 2026 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import socket -from unittest.mock import MagicMock, Mock, patch - -import pytest -import torch -import torch.distributed as dist -import torch.multiprocessing as mp - -from diffusers import ModelMixin -from diffusers.models._modeling_parallel import ContextParallelConfig, ParallelConfig, TensorParallelConfig -from diffusers.utils.distributed_utils import sync_default_rng_seed - - -def _find_free_port(): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("", 0)) - return s.getsockname()[1] - - -def _sync_seed_worker(rank, world_size, master_port, seeds, advances, return_dict): - os.environ["MASTER_ADDR"] = "localhost" - os.environ["MASTER_PORT"] = str(master_port) - dist.init_process_group(backend="gloo", rank=rank, world_size=world_size) - try: - torch.manual_seed(seeds[rank]) - torch.randn(advances[rank]) - sync_default_rng_seed(device=torch.device("cpu")) - assert torch.initial_seed() == seeds[0] - return_dict[rank] = torch.randn(4) - finally: - dist.destroy_process_group() - - -def _run(seeds, advances): - world_size = len(seeds) - return_dict = mp.Manager().dict() - mp.spawn( - _sync_seed_worker, - args=(world_size, _find_free_port(), seeds, advances, return_dict), - nprocs=world_size, - join=True, - ) - return [return_dict[rank] for rank in range(world_size)] - - -def _expected_draws(seed): - torch.manual_seed(seed) - return torch.randn(4) - - -def test_sync_default_rng_seed_aligns_different_seeds(): - draws = _run(seeds=[0, 1], advances=[1, 2]) - assert torch.equal(draws[0], _expected_draws(0)) - assert torch.equal(draws[1], _expected_draws(0)) - - -def test_sync_default_rng_seed_aligns_shared_seed_at_different_offsets(): - # Same seed, but rank 1 has drawn more random numbers, so the streams are out of step until they are reset. - draws = _run(seeds=[7, 7], advances=[1, 2]) - assert torch.equal(draws[0], _expected_draws(7)) - assert torch.equal(draws[1], _expected_draws(7)) - - -@pytest.mark.parametrize("seed", [2**63, 2**64 - 1]) -def test_sync_default_rng_seed_preserves_unsigned_seed(seed): - draws = _run(seeds=[seed, 1], advances=[1, 2]) - for draw in draws: - assert torch.equal(draw, _expected_draws(seed)) - - -def test_sync_default_rng_seed_uses_explicit_device(): - device = torch.device("cuda:1") - with ( - patch("torch.initial_seed", return_value=2**64 - 1), - patch("torch.distributed.broadcast_object_list") as broadcast, - patch("torch.manual_seed") as manual_seed, - ): - sync_default_rng_seed(device=device) - - broadcast.assert_called_once_with([2**64 - 1], src=0, group=None, device=device) - manual_seed.assert_called_once_with(2**64 - 1) - - -def test_sync_default_rng_seed_uses_subgroup_source(): - group = Mock() - device = torch.device("cpu") - with ( - patch("torch.initial_seed", return_value=7), - patch("torch.distributed.get_global_rank", return_value=2) as get_global_rank, - patch("torch.distributed.broadcast_object_list") as broadcast, - patch("torch.manual_seed"), - ): - sync_default_rng_seed(group=group, device=device) - - get_global_rank.assert_called_once_with(group, 0) - broadcast.assert_called_once_with([7], src=2, group=group, device=device) - - -@pytest.mark.parametrize("parallel_type", ["tp", "cp", "empty"]) -def test_resolve_parallel_config_syncs_world_only_for_parallel_models(parallel_type): - mesh = MagicMock() - mesh.size.return_value = 2 - mesh.ndim = 2 - mesh.mesh_dim_names = ("ring", "ulysses") - if parallel_type == "tp": - config = TensorParallelConfig(mesh=mesh) - elif parallel_type == "cp": - config = ContextParallelConfig(ring_degree=2, mesh=mesh) - else: - config = ParallelConfig() - - with ( - patch("torch.distributed.is_available", return_value=True), - patch("torch.distributed.is_initialized", return_value=True), - patch("torch.distributed.get_rank", return_value=1), - patch("torch.distributed.get_world_size", return_value=4), - patch("torch._C._get_accelerator", return_value=torch.device("cuda")), - patch("torch.accelerator.device_count", return_value=4), - patch("diffusers.models.modeling_utils.sync_default_rng_seed") as sync_seed, - ): - resolved = ModelMixin()._resolve_parallel_config(config) - - if parallel_type == "empty": - sync_seed.assert_not_called() - else: - sync_seed.assert_called_once_with(device=torch.device("cuda:1")) - assert resolved._device == torch.device("cuda:1") From 044e06ac879e8f8a605a334cb8df777d405f0d1d Mon Sep 17 00:00:00 2001 From: JingyaHuang Date: Fri, 6 Nov 2026 12:02:47 +0000 Subject: [PATCH 06/11] fix: sync default RNG only within the parallel mesh --- src/diffusers/models/modeling_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/diffusers/models/modeling_utils.py b/src/diffusers/models/modeling_utils.py index 4e95f28e4af0..9b504d376517 100644 --- a/src/diffusers/models/modeling_utils.py +++ b/src/diffusers/models/modeling_utils.py @@ -1742,7 +1742,9 @@ def _resolve_parallel_config( self._parallel_config = config if mesh is not None: - sync_default_rng_seed(device=config._device) + # Only the ranks of this mesh must draw the same numbers; replicas outside it keep their own seeds. + group = (mesh._flatten() if mesh.ndim > 1 else mesh).get_group() + sync_default_rng_seed(group=group, device=config._device) return config def enable_parallelism( From f3cb49091e450b468e63cc4b314bc1d66d727b19 Mon Sep 17 00:00:00 2001 From: JingyaHuang Date: Fri, 6 Nov 2026 12:35:51 +0000 Subject: [PATCH 07/11] test: move parallel RNG sync checks into a standalone test --- tests/models/testing_utils/parallelism.py | 106 ++++++++++++++++++---- 1 file changed, 90 insertions(+), 16 deletions(-) diff --git a/tests/models/testing_utils/parallelism.py b/tests/models/testing_utils/parallelism.py index 98a6f917380e..1a537019302c 100644 --- a/tests/models/testing_utils/parallelism.py +++ b/tests/models/testing_utils/parallelism.py @@ -20,7 +20,10 @@ import torch import torch.distributed as dist import torch.multiprocessing as mp +from torch.distributed.device_mesh import init_device_mesh +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models import ModelMixin from diffusers.models._modeling_parallel import ContextParallelConfig, TensorParallelConfig from diffusers.models.attention_dispatch import AttentionBackendName, _AttentionBackendRegistry from diffusers.utils import SAFE_WEIGHTS_INDEX_NAME @@ -31,6 +34,7 @@ is_context_parallel, is_kernels_available, is_tensor_parallel, + is_torch_neuronx_available, require_torch_multi_accelerator, require_torch_neuron, require_torch_tpu, @@ -109,16 +113,8 @@ def _context_parallel_worker( # Enable context parallelism cp_config = ContextParallelConfig(**cp_dict) - randn_tensor((rank + 1,), device=device) model.enable_parallelism(config=cp_config) - latents = randn_tensor((1, 4, 8, 8), device=device) - gathered_latents = [torch.empty_like(latents) for _ in range(world_size)] - dist.all_gather(gathered_latents, latents) - assert all(torch.equal(latents, other) for other in gathered_latents), ( - "Unseeded latents differ across context-parallel ranks." - ) - # Run forward pass with torch.no_grad(): output = model(**inputs_on_device, return_dict=False)[0] @@ -301,16 +297,8 @@ def _tensor_parallel_worker( inputs_on_device = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()} # Shard the model across all ranks; the device mesh is built from `tp_degree` on the active accelerator. - randn_tensor((rank + 1,), device=device) model.enable_parallelism(config=TensorParallelConfig(tp_degree=world_size)) - latents = randn_tensor((1, 4, 8, 8), device=device) - gathered_latents = [torch.empty_like(latents) for _ in range(world_size)] - dist.all_gather(gathered_latents, latents) - assert all(torch.equal(latents, other) for other in gathered_latents), ( - "Unseeded latents differ across tensor-parallel ranks." - ) - with torch.no_grad(): output = model(**inputs_on_device, return_dict=False)[0] @@ -895,3 +883,89 @@ def test_context_parallel_attn_backend_inference(self, cp_type, attention_backen cp_output = torch.tensor(return_dict["output"], dtype=ref_output.dtype) torch.testing.assert_close(ref_output, cp_output, atol=1e-2, rtol=1e-2) + + +def _parallel_device_type(): + """The accelerator type to run two-rank parallelism tests on, or `None` when there are fewer than two.""" + if torch.cuda.device_count() > 1: + return "cuda" + if torch.xpu.device_count() > 1: + return "xpu" + if is_torch_neuronx_available() and hasattr(torch, "neuron") and torch.neuron.is_available(): + return "neuron" + return None + + +class _ParallelRNGDummyModel(ModelMixin, ConfigMixin): + """The smallest model `enable_parallelism` accepts: one shardable `Linear` and an empty context-parallel plan.""" + + _tp_plan = {"proj": "colwise"} + _cp_plan = {} + + @register_to_config + def __init__(self): + super().__init__() + self.proj = torch.nn.Linear(4, 4) + + +def _parallel_rng_sync_worker(rank, world_size, master_port, device_type, parallelism, return_dict): + """Seed every rank differently, enable `parallelism`, and report whether all ranks then draw the same latents.""" + try: + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + + device_config = DEVICE_CONFIG[device_type] + dist.init_process_group(backend=device_config["backend"], rank=rank, world_size=world_size) + device_config["module"].set_device(rank) + device = torch.device(f"{device_type}:{rank}") + model = _ParallelRNGDummyModel().to(device) + + if parallelism == "tp": + config = TensorParallelConfig(tp_degree=world_size) + elif parallelism == "cp": + config = ContextParallelConfig(ulysses_degree=world_size) + else: + # Every rank is its own one-rank TP group, i.e. an independent replica. + mesh = init_device_mesh(device_type, mesh_shape=(world_size, 1), mesh_dim_names=("dp", "tp")) + config = TensorParallelConfig(mesh=mesh["tp"]) + + torch.manual_seed(rank) + model.enable_parallelism(config=config) + + latents = randn_tensor((1, 4, 8, 8), device=device) + gathered_latents = [torch.empty_like(latents) for _ in range(world_size)] + dist.all_gather(gathered_latents, latents) + if rank == 0: + return_dict["same_latents"] = all(torch.equal(latents, other) for other in gathered_latents) + + except Exception as e: + if rank == 0: + return_dict["error"] = str(e) + finally: + if dist.is_initialized(): + dist.destroy_process_group() + + +@is_tensor_parallel +@is_context_parallel +@pytest.mark.skipif(_parallel_device_type() is None, reason="test requires two accelerators") +class TestParallelRNGSync: + """`enable_parallelism` gives every rank of the parallel mesh the same default RNG, and only those ranks.""" + + @pytest.mark.parametrize( + "parallelism,expect_same", + [("tp", True), ("cp", True), ("tp_submesh", False)], + ids=["tensor_parallel", "context_parallel", "tensor_parallel_submesh"], + ) + def test_enable_parallelism_syncs_rng_within_mesh(self, parallelism, expect_same): + world_size = 2 + return_dict = mp.Manager().dict() + mp.spawn( + _parallel_rng_sync_worker, + args=(world_size, _find_free_port(), _parallel_device_type(), parallelism, return_dict), + nprocs=world_size, + join=True, + ) + + assert "error" not in return_dict, f"RNG sync worker failed: {return_dict.get('error')}" + assert return_dict["same_latents"] == expect_same From a0330c8f96251a2705e3d346aae4ce1334d37f90 Mon Sep 17 00:00:00 2001 From: JingyaHuang Date: Mon, 9 Nov 2026 17:40:51 +0000 Subject: [PATCH 08/11] review: test RNG sync against non-parallel run and from_pretrained --- src/diffusers/models/modeling_utils.py | 2 +- tests/models/test_modeling_parallel.py | 212 ++++++++++++++++++++++ tests/models/testing_utils/parallelism.py | 91 ---------- tests/testing_utils.py | 28 ++- 4 files changed, 239 insertions(+), 94 deletions(-) create mode 100644 tests/models/test_modeling_parallel.py diff --git a/src/diffusers/models/modeling_utils.py b/src/diffusers/models/modeling_utils.py index 2da6b2536ced..98cd398df4ca 100644 --- a/src/diffusers/models/modeling_utils.py +++ b/src/diffusers/models/modeling_utils.py @@ -1242,7 +1242,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike | None tp_config = ( parallel_config if isinstance(parallel_config, TensorParallelConfig) - else parallel_config.tensor_parallel_config + else getattr(parallel_config, "tensor_parallel_config", None) ) if tp_config is not None and tp_config.tp_degree == 1 and tp_config.mesh is None: # Nothing to shard, so take the ordinary loader rather than building 1-rank DTensors. diff --git a/tests/models/test_modeling_parallel.py b/tests/models/test_modeling_parallel.py new file mode 100644 index 000000000000..b2f09d9e4805 --- /dev/null +++ b/tests/models/test_modeling_parallel.py @@ -0,0 +1,212 @@ +# coding=utf-8 +# Copyright 2026 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import os + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from torch.distributed.device_mesh import init_device_mesh + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models import ModelMixin +from diffusers.models._modeling_parallel import ( + ContextParallelConfig, + ContextParallelInput, + ContextParallelOutput, + TensorParallelConfig, +) +from diffusers.utils.torch_utils import randn_tensor + +from ..testing_utils import ( + is_context_parallel, + is_tensor_parallel, + is_torch_tpu_available, + require_torch_multi_accelerator, + torch_device, +) +from .testing_utils.parallelism import DEVICE_CONFIG, _find_free_port + + +class _ParallelRNGDummyModel(ModelMixin, ConfigMixin): + """A two-`Linear` MLP: tensor parallelism shards it column- then row-wise, context parallelism splits its sequence.""" + + _tp_plan = {"proj_in": "colwise", "proj_out": "rowwise"} + _cp_plan = { + "": {"hidden_states": ContextParallelInput(split_dim=1, expected_dims=3)}, + "proj_out": ContextParallelOutput(gather_dim=1, expected_dims=3), + } + + @register_to_config + def __init__(self, dim: int = 8): + super().__init__() + self.proj_in = torch.nn.Linear(dim, 4 * dim) + self.proj_out = torch.nn.Linear(4 * dim, dim) + + def forward(self, hidden_states): + return self.proj_out(torch.nn.functional.gelu(self.proj_in(hidden_states))) + + +_PARALLEL_RNG_LATENTS_SHAPE = (1, 8, 8) + + +def _parallel_rng_device_type(): + """The accelerator type to run on. `torch_device` stays "cpu" on a TPU host unless `DIFFUSERS_TEST_DEVICE` is set.""" + device_type = torch.device(torch_device).type + if device_type == "cpu" and is_torch_tpu_available(): + return "tpu" + return device_type + + +def _parallel_rng_device(device_type, rank): + if device_type == "tpu": + # Each spawned process is bound to one chip. Avoid bf16 matmuls to keep the tolerance tight. + torch.set_float32_matmul_precision("highest") + return torch.device("tpu") + DEVICE_CONFIG[device_type]["module"].set_device(rank) + return torch.device(f"{device_type}:{rank}") + + +@contextlib.contextmanager +def _parallel_rng_spawn_env(device_type, world_size): + """Prepare the environment `mp.spawn` needs to start `world_size` workers on `device_type`.""" + if device_type != "tpu": + yield + return + + from torch_tpu._internal.distributed.launchers.singlehost_wrapper import prepare_tpu_environment + + tpu_env = ("TORCH_TPU_TOPOLOGY", "TORCH_TPU_SLICEBUILDER_ADDRESSES") + for key in tpu_env: + os.environ.pop(key, None) + prepare_tpu_environment(world_size) + try: + yield + finally: + for key in tpu_env: + os.environ.pop(key, None) + + +def _parallel_rng_reference_worker(rank, device_type, checkpoint_dir, return_dict): + """Single-device run seeded like rank 0 of the parallel run: the latents and output parallelism must reproduce.""" + try: + device = _parallel_rng_device(device_type, 0) + model = _ParallelRNGDummyModel.from_pretrained(checkpoint_dir).to(device) + + torch.manual_seed(0) + latents = randn_tensor(_PARALLEL_RNG_LATENTS_SHAPE, device=device) + with torch.no_grad(): + output = model(latents) + return_dict["latents"] = latents.cpu().tolist() + return_dict["output"] = output.cpu().tolist() + except Exception as e: + return_dict["error"] = str(e) + + +def _parallel_rng_sync_worker( + rank, world_size, master_port, device_type, parallelism, loading, checkpoint_dir, return_dict +): + """Seed every rank differently, load the model with `parallelism`, and report the latents each rank then draws.""" + try: + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + + dist.init_process_group(backend=DEVICE_CONFIG[device_type]["backend"], rank=rank, world_size=world_size) + device = _parallel_rng_device(device_type, rank) + + if parallelism == "tp": + config = TensorParallelConfig(tp_degree=world_size) + elif parallelism == "cp": + config = ContextParallelConfig(ulysses_degree=world_size) + else: + # Every rank is its own one-rank TP group, i.e. an independent replica. + mesh = init_device_mesh(device_type, mesh_shape=(world_size, 1), mesh_dim_names=("dp", "tp")) + config = TensorParallelConfig(mesh=mesh["tp"]) + + torch.manual_seed(rank) + if loading == "from_pretrained": + model = _ParallelRNGDummyModel.from_pretrained(checkpoint_dir, parallel_config=config).to(device) + else: + model = _ParallelRNGDummyModel.from_pretrained(checkpoint_dir).to(device) + model.enable_parallelism(config=config) + + latents = randn_tensor(_PARALLEL_RNG_LATENTS_SHAPE, device=device) + with torch.no_grad(): + output = model(latents) + + gathered_latents = [torch.empty_like(latents) for _ in range(world_size)] + dist.all_gather(gathered_latents, latents) + if rank == 0: + return_dict["same_latents"] = all(torch.equal(latents, other) for other in gathered_latents) + return_dict["latents"] = latents.cpu().tolist() + return_dict["output"] = output.cpu().tolist() + + except Exception as e: + if rank == 0: + return_dict["error"] = str(e) + finally: + if dist.is_initialized(): + dist.destroy_process_group() + + +@is_tensor_parallel +@is_context_parallel +@require_torch_multi_accelerator(include_neuron=True, include_tpu=True) +class TestParallelRNGSync: + """Loading with a parallel config gives every rank of the parallel mesh the same default RNG, and only those ranks. + + Rank 0 of the parallel run must also draw the latents, and produce the output, of a non-parallel run with its seed. + """ + + @pytest.mark.parametrize("loading", ["enable_parallelism", "from_pretrained"]) + @pytest.mark.parametrize( + "parallelism,expect_same", + [("tp", True), ("cp", True), ("tp_submesh", False)], + ids=["tensor_parallel", "context_parallel", "tensor_parallel_submesh"], + ) + def test_parallelism_syncs_rng_within_mesh(self, parallelism, expect_same, loading, tmp_path): + device_type = _parallel_rng_device_type() + # Smallest slice every TPU generation supports, as in `TensorParallelTPUTesterMixin`. + world_size = 4 if device_type == "tpu" else 2 + if device_type == "tpu": + from torch_tpu._internal.utils import hardware + + if hardware.get_tpu_device_count() < world_size: + pytest.skip(f"Needs at least {world_size} TPU chips.") + + _ParallelRNGDummyModel().save_pretrained(tmp_path) + + reference = mp.Manager().dict() + with _parallel_rng_spawn_env(device_type, 1): + mp.spawn(_parallel_rng_reference_worker, args=(device_type, tmp_path, reference), nprocs=1, join=True) + assert "error" not in reference, f"RNG reference worker failed: {reference.get('error')}" + + return_dict = mp.Manager().dict() + with _parallel_rng_spawn_env(device_type, world_size): + mp.spawn( + _parallel_rng_sync_worker, + args=(world_size, _find_free_port(), device_type, parallelism, loading, tmp_path, return_dict), + nprocs=world_size, + join=True, + ) + + assert "error" not in return_dict, f"RNG sync worker failed: {return_dict.get('error')}" + assert return_dict["same_latents"] == expect_same + assert torch.equal(torch.tensor(return_dict["latents"]), torch.tensor(reference["latents"])) + torch.testing.assert_close( + torch.tensor(return_dict["output"]), torch.tensor(reference["output"]), atol=1e-2, rtol=1e-2 + ) diff --git a/tests/models/testing_utils/parallelism.py b/tests/models/testing_utils/parallelism.py index 1a537019302c..de19a7edd212 100644 --- a/tests/models/testing_utils/parallelism.py +++ b/tests/models/testing_utils/parallelism.py @@ -20,21 +20,16 @@ import torch import torch.distributed as dist import torch.multiprocessing as mp -from torch.distributed.device_mesh import init_device_mesh -from diffusers.configuration_utils import ConfigMixin, register_to_config -from diffusers.models import ModelMixin from diffusers.models._modeling_parallel import ContextParallelConfig, TensorParallelConfig from diffusers.models.attention_dispatch import AttentionBackendName, _AttentionBackendRegistry from diffusers.utils import SAFE_WEIGHTS_INDEX_NAME -from diffusers.utils.torch_utils import randn_tensor from ...testing_utils import ( is_attention, is_context_parallel, is_kernels_available, is_tensor_parallel, - is_torch_neuronx_available, require_torch_multi_accelerator, require_torch_neuron, require_torch_tpu, @@ -883,89 +878,3 @@ def test_context_parallel_attn_backend_inference(self, cp_type, attention_backen cp_output = torch.tensor(return_dict["output"], dtype=ref_output.dtype) torch.testing.assert_close(ref_output, cp_output, atol=1e-2, rtol=1e-2) - - -def _parallel_device_type(): - """The accelerator type to run two-rank parallelism tests on, or `None` when there are fewer than two.""" - if torch.cuda.device_count() > 1: - return "cuda" - if torch.xpu.device_count() > 1: - return "xpu" - if is_torch_neuronx_available() and hasattr(torch, "neuron") and torch.neuron.is_available(): - return "neuron" - return None - - -class _ParallelRNGDummyModel(ModelMixin, ConfigMixin): - """The smallest model `enable_parallelism` accepts: one shardable `Linear` and an empty context-parallel plan.""" - - _tp_plan = {"proj": "colwise"} - _cp_plan = {} - - @register_to_config - def __init__(self): - super().__init__() - self.proj = torch.nn.Linear(4, 4) - - -def _parallel_rng_sync_worker(rank, world_size, master_port, device_type, parallelism, return_dict): - """Seed every rank differently, enable `parallelism`, and report whether all ranks then draw the same latents.""" - try: - os.environ["MASTER_ADDR"] = "localhost" - os.environ["MASTER_PORT"] = str(master_port) - - device_config = DEVICE_CONFIG[device_type] - dist.init_process_group(backend=device_config["backend"], rank=rank, world_size=world_size) - device_config["module"].set_device(rank) - device = torch.device(f"{device_type}:{rank}") - model = _ParallelRNGDummyModel().to(device) - - if parallelism == "tp": - config = TensorParallelConfig(tp_degree=world_size) - elif parallelism == "cp": - config = ContextParallelConfig(ulysses_degree=world_size) - else: - # Every rank is its own one-rank TP group, i.e. an independent replica. - mesh = init_device_mesh(device_type, mesh_shape=(world_size, 1), mesh_dim_names=("dp", "tp")) - config = TensorParallelConfig(mesh=mesh["tp"]) - - torch.manual_seed(rank) - model.enable_parallelism(config=config) - - latents = randn_tensor((1, 4, 8, 8), device=device) - gathered_latents = [torch.empty_like(latents) for _ in range(world_size)] - dist.all_gather(gathered_latents, latents) - if rank == 0: - return_dict["same_latents"] = all(torch.equal(latents, other) for other in gathered_latents) - - except Exception as e: - if rank == 0: - return_dict["error"] = str(e) - finally: - if dist.is_initialized(): - dist.destroy_process_group() - - -@is_tensor_parallel -@is_context_parallel -@pytest.mark.skipif(_parallel_device_type() is None, reason="test requires two accelerators") -class TestParallelRNGSync: - """`enable_parallelism` gives every rank of the parallel mesh the same default RNG, and only those ranks.""" - - @pytest.mark.parametrize( - "parallelism,expect_same", - [("tp", True), ("cp", True), ("tp_submesh", False)], - ids=["tensor_parallel", "context_parallel", "tensor_parallel_submesh"], - ) - def test_enable_parallelism_syncs_rng_within_mesh(self, parallelism, expect_same): - world_size = 2 - return_dict = mp.Manager().dict() - mp.spawn( - _parallel_rng_sync_worker, - args=(world_size, _find_free_port(), _parallel_device_type(), parallelism, return_dict), - nprocs=world_size, - join=True, - ) - - assert "error" not in return_dict, f"RNG sync worker failed: {return_dict.get('error')}" - assert return_dict["same_latents"] == expect_same diff --git a/tests/testing_utils.py b/tests/testing_utils.py index cda114cbde6d..914c1c36e0c2 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -597,18 +597,42 @@ def require_torch_multi_gpu(test_case): return pytest.mark.skipif(torch.cuda.device_count() <= 1, reason="test requires multiple GPUs")(test_case) -def require_torch_multi_accelerator(test_case): +def require_torch_multi_accelerator(test_case=None, *, include_neuron: bool = False, include_tpu: bool = False): """ Decorator marking a test that requires a multi-accelerator setup (in PyTorch). These tests are skipped on a machine without multiple hardware accelerators. + + Neuron devices and TPU chips only count with `include_neuron=True` and `include_tpu=True` respectively, because some + multi-accelerator tester mixins do not run on them yet. """ + if test_case is None: + return functools.partial( + require_torch_multi_accelerator, include_neuron=include_neuron, include_tpu=include_tpu + ) + if not is_torch_available(): return pytest.mark.skip(reason="test requires PyTorch")(test_case) import torch + neuron_device_count = ( + torch.neuron.device_count() + if include_neuron and is_torch_neuronx_available() and hasattr(torch, "neuron") and torch.neuron.is_available() + else 0 + ) + tpu_device_count = 0 + if include_tpu and is_torch_tpu_available(): + from torch_tpu._internal.utils import hardware + + tpu_device_count = hardware.get_tpu_device_count() + return pytest.mark.skipif( - not (torch.cuda.device_count() > 1 or torch.xpu.device_count() > 1), + not ( + torch.cuda.device_count() > 1 + or torch.xpu.device_count() > 1 + or neuron_device_count > 1 + or tpu_device_count > 1 + ), reason="test requires multiple hardware accelerators", )(test_case) From afe646e53b737ea13f44c907f47badffeefd0389 Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Tue, 10 Nov 2026 03:50:08 +0000 Subject: [PATCH 09/11] generalize the test case for sync rng for all supported models. --- tests/models/test_modeling_parallel.py | 212 ---------------------- tests/models/testing_utils/parallelism.py | 211 +++++++++++++++++++++ tests/testing_utils.py | 28 +-- 3 files changed, 213 insertions(+), 238 deletions(-) delete mode 100644 tests/models/test_modeling_parallel.py diff --git a/tests/models/test_modeling_parallel.py b/tests/models/test_modeling_parallel.py deleted file mode 100644 index b2f09d9e4805..000000000000 --- a/tests/models/test_modeling_parallel.py +++ /dev/null @@ -1,212 +0,0 @@ -# coding=utf-8 -# Copyright 2026 HuggingFace Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import contextlib -import os - -import pytest -import torch -import torch.distributed as dist -import torch.multiprocessing as mp -from torch.distributed.device_mesh import init_device_mesh - -from diffusers.configuration_utils import ConfigMixin, register_to_config -from diffusers.models import ModelMixin -from diffusers.models._modeling_parallel import ( - ContextParallelConfig, - ContextParallelInput, - ContextParallelOutput, - TensorParallelConfig, -) -from diffusers.utils.torch_utils import randn_tensor - -from ..testing_utils import ( - is_context_parallel, - is_tensor_parallel, - is_torch_tpu_available, - require_torch_multi_accelerator, - torch_device, -) -from .testing_utils.parallelism import DEVICE_CONFIG, _find_free_port - - -class _ParallelRNGDummyModel(ModelMixin, ConfigMixin): - """A two-`Linear` MLP: tensor parallelism shards it column- then row-wise, context parallelism splits its sequence.""" - - _tp_plan = {"proj_in": "colwise", "proj_out": "rowwise"} - _cp_plan = { - "": {"hidden_states": ContextParallelInput(split_dim=1, expected_dims=3)}, - "proj_out": ContextParallelOutput(gather_dim=1, expected_dims=3), - } - - @register_to_config - def __init__(self, dim: int = 8): - super().__init__() - self.proj_in = torch.nn.Linear(dim, 4 * dim) - self.proj_out = torch.nn.Linear(4 * dim, dim) - - def forward(self, hidden_states): - return self.proj_out(torch.nn.functional.gelu(self.proj_in(hidden_states))) - - -_PARALLEL_RNG_LATENTS_SHAPE = (1, 8, 8) - - -def _parallel_rng_device_type(): - """The accelerator type to run on. `torch_device` stays "cpu" on a TPU host unless `DIFFUSERS_TEST_DEVICE` is set.""" - device_type = torch.device(torch_device).type - if device_type == "cpu" and is_torch_tpu_available(): - return "tpu" - return device_type - - -def _parallel_rng_device(device_type, rank): - if device_type == "tpu": - # Each spawned process is bound to one chip. Avoid bf16 matmuls to keep the tolerance tight. - torch.set_float32_matmul_precision("highest") - return torch.device("tpu") - DEVICE_CONFIG[device_type]["module"].set_device(rank) - return torch.device(f"{device_type}:{rank}") - - -@contextlib.contextmanager -def _parallel_rng_spawn_env(device_type, world_size): - """Prepare the environment `mp.spawn` needs to start `world_size` workers on `device_type`.""" - if device_type != "tpu": - yield - return - - from torch_tpu._internal.distributed.launchers.singlehost_wrapper import prepare_tpu_environment - - tpu_env = ("TORCH_TPU_TOPOLOGY", "TORCH_TPU_SLICEBUILDER_ADDRESSES") - for key in tpu_env: - os.environ.pop(key, None) - prepare_tpu_environment(world_size) - try: - yield - finally: - for key in tpu_env: - os.environ.pop(key, None) - - -def _parallel_rng_reference_worker(rank, device_type, checkpoint_dir, return_dict): - """Single-device run seeded like rank 0 of the parallel run: the latents and output parallelism must reproduce.""" - try: - device = _parallel_rng_device(device_type, 0) - model = _ParallelRNGDummyModel.from_pretrained(checkpoint_dir).to(device) - - torch.manual_seed(0) - latents = randn_tensor(_PARALLEL_RNG_LATENTS_SHAPE, device=device) - with torch.no_grad(): - output = model(latents) - return_dict["latents"] = latents.cpu().tolist() - return_dict["output"] = output.cpu().tolist() - except Exception as e: - return_dict["error"] = str(e) - - -def _parallel_rng_sync_worker( - rank, world_size, master_port, device_type, parallelism, loading, checkpoint_dir, return_dict -): - """Seed every rank differently, load the model with `parallelism`, and report the latents each rank then draws.""" - try: - os.environ["MASTER_ADDR"] = "localhost" - os.environ["MASTER_PORT"] = str(master_port) - - dist.init_process_group(backend=DEVICE_CONFIG[device_type]["backend"], rank=rank, world_size=world_size) - device = _parallel_rng_device(device_type, rank) - - if parallelism == "tp": - config = TensorParallelConfig(tp_degree=world_size) - elif parallelism == "cp": - config = ContextParallelConfig(ulysses_degree=world_size) - else: - # Every rank is its own one-rank TP group, i.e. an independent replica. - mesh = init_device_mesh(device_type, mesh_shape=(world_size, 1), mesh_dim_names=("dp", "tp")) - config = TensorParallelConfig(mesh=mesh["tp"]) - - torch.manual_seed(rank) - if loading == "from_pretrained": - model = _ParallelRNGDummyModel.from_pretrained(checkpoint_dir, parallel_config=config).to(device) - else: - model = _ParallelRNGDummyModel.from_pretrained(checkpoint_dir).to(device) - model.enable_parallelism(config=config) - - latents = randn_tensor(_PARALLEL_RNG_LATENTS_SHAPE, device=device) - with torch.no_grad(): - output = model(latents) - - gathered_latents = [torch.empty_like(latents) for _ in range(world_size)] - dist.all_gather(gathered_latents, latents) - if rank == 0: - return_dict["same_latents"] = all(torch.equal(latents, other) for other in gathered_latents) - return_dict["latents"] = latents.cpu().tolist() - return_dict["output"] = output.cpu().tolist() - - except Exception as e: - if rank == 0: - return_dict["error"] = str(e) - finally: - if dist.is_initialized(): - dist.destroy_process_group() - - -@is_tensor_parallel -@is_context_parallel -@require_torch_multi_accelerator(include_neuron=True, include_tpu=True) -class TestParallelRNGSync: - """Loading with a parallel config gives every rank of the parallel mesh the same default RNG, and only those ranks. - - Rank 0 of the parallel run must also draw the latents, and produce the output, of a non-parallel run with its seed. - """ - - @pytest.mark.parametrize("loading", ["enable_parallelism", "from_pretrained"]) - @pytest.mark.parametrize( - "parallelism,expect_same", - [("tp", True), ("cp", True), ("tp_submesh", False)], - ids=["tensor_parallel", "context_parallel", "tensor_parallel_submesh"], - ) - def test_parallelism_syncs_rng_within_mesh(self, parallelism, expect_same, loading, tmp_path): - device_type = _parallel_rng_device_type() - # Smallest slice every TPU generation supports, as in `TensorParallelTPUTesterMixin`. - world_size = 4 if device_type == "tpu" else 2 - if device_type == "tpu": - from torch_tpu._internal.utils import hardware - - if hardware.get_tpu_device_count() < world_size: - pytest.skip(f"Needs at least {world_size} TPU chips.") - - _ParallelRNGDummyModel().save_pretrained(tmp_path) - - reference = mp.Manager().dict() - with _parallel_rng_spawn_env(device_type, 1): - mp.spawn(_parallel_rng_reference_worker, args=(device_type, tmp_path, reference), nprocs=1, join=True) - assert "error" not in reference, f"RNG reference worker failed: {reference.get('error')}" - - return_dict = mp.Manager().dict() - with _parallel_rng_spawn_env(device_type, world_size): - mp.spawn( - _parallel_rng_sync_worker, - args=(world_size, _find_free_port(), device_type, parallelism, loading, tmp_path, return_dict), - nprocs=world_size, - join=True, - ) - - assert "error" not in return_dict, f"RNG sync worker failed: {return_dict.get('error')}" - assert return_dict["same_latents"] == expect_same - assert torch.equal(torch.tensor(return_dict["latents"]), torch.tensor(reference["latents"])) - torch.testing.assert_close( - torch.tensor(return_dict["output"]), torch.tensor(reference["output"]), atol=1e-2, rtol=1e-2 - ) diff --git a/tests/models/testing_utils/parallelism.py b/tests/models/testing_utils/parallelism.py index de19a7edd212..56dacebb5575 100644 --- a/tests/models/testing_utils/parallelism.py +++ b/tests/models/testing_utils/parallelism.py @@ -24,6 +24,7 @@ from diffusers.models._modeling_parallel import ContextParallelConfig, TensorParallelConfig from diffusers.models.attention_dispatch import AttentionBackendName, _AttentionBackendRegistry from diffusers.utils import SAFE_WEIGHTS_INDEX_NAME +from diffusers.utils.torch_utils import randn_tensor from ...testing_utils import ( is_attention, @@ -382,6 +383,158 @@ def _tensor_parallel_from_pretrained_worker( dist.destroy_process_group() +def _parallel_rng_sync_worker( + rank, + world_size, + master_port, + model_class, + checkpoint_dir, + inputs_dict, + main_input_name, + parallelism, + loading, + return_dict, + device_type=torch_device, +): + """Worker for the default RNG sync that loading with a parallel config does across the ranks of its mesh. + + Each rank seeds itself differently, loads the model with `parallelism`, then draws the main input from the default + RNG and runs a forward pass. Rank 0 reports whether every rank drew the same input, along with its own input and + output so the caller can compare against a single-device run on that input. + """ + try: + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + os.environ["RANK"] = str(rank) + os.environ["LOCAL_RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world_size) + + device_config = DEVICE_CONFIG.get(device_type, DEVICE_CONFIG["cuda"]) + dist.init_process_group(backend=device_config["backend"], rank=rank, world_size=world_size) + + if device_type == "tpu": + # Each spawned process is bound to one chip. Avoid bf16 matmuls to keep the tolerance tight. + device = torch.device("tpu") + torch.set_float32_matmul_precision("highest") + else: + device_config["module"].set_device(rank) + device = torch.device(f"{device_type}:{rank}") + + from torch.distributed.tensor import DTensor + + if parallelism == "tensor_parallel": + config = TensorParallelConfig(tp_degree=world_size) + elif parallelism == "context_parallel": + config = ContextParallelConfig(ulysses_degree=world_size) + else: + # Every rank is its own one-rank TP group, i.e. an independent replica. + mesh = torch.distributed.device_mesh.init_device_mesh( + device_type, mesh_shape=(world_size, 1), mesh_dim_names=("dp", "tp") + ) + config = TensorParallelConfig(mesh=mesh["tp"]) + + torch.manual_seed(rank) + if loading == "from_pretrained": + model = model_class.from_pretrained(checkpoint_dir, parallel_config=config).to(device) + else: + model = model_class.from_pretrained(checkpoint_dir).to(device) + model.enable_parallelism(config=config) + model.eval() + + inputs_on_device = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()} + noise = randn_tensor( + inputs_on_device[main_input_name].shape, device=device, dtype=inputs_on_device[main_input_name].dtype + ) + inputs_on_device[main_input_name] = noise + with torch.no_grad(): + output = model(**inputs_on_device, return_dict=False)[0] + if isinstance(output, DTensor): + output = output.full_tensor() + + gathered_noise = [torch.empty_like(noise) for _ in range(world_size)] + dist.all_gather(gathered_noise, noise) + if rank == 0: + return_dict["status"] = "success" + return_dict["same_noise"] = all(torch.equal(noise, other) for other in gathered_noise) + # Redraw from rank 0's seed, which is what a non-parallel run with that seed would draw. + torch.manual_seed(rank) + seeded_noise = randn_tensor(noise.shape, device=device, dtype=noise.dtype) + return_dict["rank_0_seed_kept"] = torch.equal(noise, seeded_noise) + # Serialise via nested list so the manager dict can transport it across processes. + return_dict["noise"] = noise.float().cpu().tolist() + return_dict["output"] = output.float().cpu().tolist() + + except Exception as e: + if rank == 0: + return_dict["status"] = "error" + return_dict["error"] = f"{type(e).__name__}: {e}" + finally: + if dist.is_initialized(): + dist.destroy_process_group() + + +def _check_parallel_rng_sync( + tester, inputs_dict, tmp_path, parallelism, loading, world_size=2, device_type=torch_device, atol=1e-3, rtol=1e-3 +): + """Check that loading `tester.model_class` with `parallelism` syncs the default RNG across the ranks of its mesh. + + The ranks start from different seeds and draw the main input after loading. They must all draw the same one, + unless each rank is an independent replica (`"tensor_parallel_submesh"`), and rank 0's output must match a + single-device run on its input. That run uses the device `inputs_dict` is on. + """ + if not torch.distributed.is_available(): + pytest.skip("torch.distributed is not available.") + + model_class = tester.model_class + plan_name = "_cp_plan" if parallelism == "context_parallel" else "_tp_plan" + if getattr(model_class, plan_name, None) is None: + pytest.skip(f"Model does not define a `{plan_name}`.") + + init_dict = tester.get_init_dict() + num_heads = init_dict.get("num_attention_heads") + if parallelism == "tensor_parallel" and num_heads is not None and num_heads % world_size != 0: + pytest.skip(f"`num_attention_heads` ({num_heads}) is not divisible by tp_degree ({world_size}).") + + main_input_name = tester.main_input_name + reference_device = inputs_dict[main_input_name].device + model = model_class(**init_dict).eval() + checkpoint_dir = str(tmp_path / "checkpoint") + model.save_pretrained(checkpoint_dir) + + return_dict = mp.Manager().dict() + mp.spawn( + _parallel_rng_sync_worker, + args=( + world_size, + _find_free_port(), + model_class, + checkpoint_dir, + {k: v.cpu() if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()}, + main_input_name, + parallelism, + loading, + return_dict, + device_type, + ), + nprocs=world_size, + join=True, + ) + assert return_dict.get("status") == "success", ( + f"Parallel RNG sync failed: {return_dict.get('error', 'Unknown error')}" + ) + # Independent replicas are outside each other's mesh, so they must keep their own seeds. + expect_same_noise = parallelism != "tensor_parallel_submesh" + assert return_dict["same_noise"] == expect_same_noise, ( + f"Ranks drew {'different' if expect_same_noise else 'the same'} inputs from the default RNG." + ) + assert return_dict["rank_0_seed_kept"], "Rank 0 did not draw the input a non-parallel run with its seed draws." + + noise = torch.tensor(return_dict["noise"], dtype=inputs_dict[main_input_name].dtype, device=reference_device) + with torch.no_grad(): + reference = model.to(reference_device)(**{**inputs_dict, main_input_name: noise}, return_dict=False)[0] + torch.testing.assert_close(reference.float().cpu(), torch.tensor(return_dict["output"]), atol=atol, rtol=rtol) + + @is_tensor_parallel @require_torch_multi_accelerator class TensorParallelTesterMixin: @@ -494,6 +647,12 @@ def test_tensor_parallel_from_pretrained(self, tmp_path, sharded): # Sharded matmuls + all-reduce reorder the summation, so allow a small tolerance over the reference. torch.testing.assert_close(reference, torch.tensor(return_dict["output"]), atol=1e-3, rtol=1e-3) + @pytest.mark.parametrize("loading", ["enable_parallelism", "from_pretrained"]) + @pytest.mark.parametrize("parallelism", ["tensor_parallel", "tensor_parallel_submesh"], ids=["mesh", "submesh"]) + def test_tensor_parallel_rng_sync(self, tmp_path, parallelism, loading): + """Loading with a TP config gives every rank of the TP mesh the same default RNG, and only those ranks.""" + _check_parallel_rng_sync(self, self.get_dummy_inputs(), tmp_path, parallelism, loading) + @is_tensor_parallel @require_torch_tpu @@ -559,6 +718,36 @@ def test_tensor_parallel_tpu_inference(self, atol=1e-3, rtol=1e-3): tp_output = torch.tensor(return_dict["output"]) torch.testing.assert_close(ref_output, tp_output, atol=atol, rtol=rtol) + def test_tensor_parallel_tpu_rng_sync(self, tmp_path, atol=2e-2, rtol=2e-2): + """Enabling TP gives every rank of the TP mesh the same default RNG.""" + from torch_tpu._internal.distributed.launchers.singlehost_wrapper import prepare_tpu_environment + from torch_tpu._internal.utils import hardware + + world_size = self.tp_world_size + if hardware.get_tpu_device_count() < world_size: + pytest.skip(f"Needs at least {world_size} TPU chips.") + + tpu_env = ("TORCH_TPU_TOPOLOGY", "TORCH_TPU_SLICEBUILDER_ADDRESSES") + for key in tpu_env: + os.environ.pop(key, None) + prepare_tpu_environment(world_size) + try: + # Inputs, and so the reference, stay on CPU: touching the TPU here would bind every chip to this process. + _check_parallel_rng_sync( + self, + self.get_dummy_inputs(device="cpu"), + tmp_path, + "tensor_parallel", + "enable_parallelism", + world_size=world_size, + device_type="tpu", + atol=atol, + rtol=rtol, + ) + finally: + for key in tpu_env: + os.environ.pop(key, None) + @is_tensor_parallel @require_torch_neuron @@ -612,6 +801,21 @@ def test_tensor_parallel_neuron_inference(self, atol=2e-2, rtol=2e-2): # Neuron runs matmuls in bf16 internally, so compare with a bf16-level tolerance. torch.testing.assert_close(ref_output, tp_output, atol=atol, rtol=rtol) + def test_tensor_parallel_neuron_rng_sync(self, tmp_path, atol=2e-2, rtol=2e-2): + """Enabling TP gives every rank of the TP mesh the same default RNG.""" + # Inputs, and so the reference, stay on CPU: touching the Neuron device here would break the spawned workers. + _check_parallel_rng_sync( + self, + self.get_dummy_inputs(device="cpu"), + tmp_path, + "tensor_parallel", + "enable_parallelism", + world_size=self.tp_world_size, + device_type="neuron", + atol=atol, + rtol=rtol, + ) + @is_context_parallel @require_torch_multi_accelerator @@ -774,6 +978,13 @@ def test_context_parallel_custom_mesh(self, cp_type, mesh_shape, mesh_dim_names) f"Custom mesh context parallel inference failed: {return_dict.get('error', 'Unknown error')}" ) + @pytest.mark.parametrize("loading", ["enable_parallelism", "from_pretrained"]) + def test_context_parallel_rng_sync(self, tmp_path, loading): + """Loading with a CP config gives every rank of the CP mesh the same default RNG.""" + _check_parallel_rng_sync( + self, self.get_dummy_inputs(), tmp_path, "context_parallel", loading, atol=1e-4, rtol=1e-4 + ) + @is_attention @is_context_parallel diff --git a/tests/testing_utils.py b/tests/testing_utils.py index 914c1c36e0c2..cda114cbde6d 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -597,42 +597,18 @@ def require_torch_multi_gpu(test_case): return pytest.mark.skipif(torch.cuda.device_count() <= 1, reason="test requires multiple GPUs")(test_case) -def require_torch_multi_accelerator(test_case=None, *, include_neuron: bool = False, include_tpu: bool = False): +def require_torch_multi_accelerator(test_case): """ Decorator marking a test that requires a multi-accelerator setup (in PyTorch). These tests are skipped on a machine without multiple hardware accelerators. - - Neuron devices and TPU chips only count with `include_neuron=True` and `include_tpu=True` respectively, because some - multi-accelerator tester mixins do not run on them yet. """ - if test_case is None: - return functools.partial( - require_torch_multi_accelerator, include_neuron=include_neuron, include_tpu=include_tpu - ) - if not is_torch_available(): return pytest.mark.skip(reason="test requires PyTorch")(test_case) import torch - neuron_device_count = ( - torch.neuron.device_count() - if include_neuron and is_torch_neuronx_available() and hasattr(torch, "neuron") and torch.neuron.is_available() - else 0 - ) - tpu_device_count = 0 - if include_tpu and is_torch_tpu_available(): - from torch_tpu._internal.utils import hardware - - tpu_device_count = hardware.get_tpu_device_count() - return pytest.mark.skipif( - not ( - torch.cuda.device_count() > 1 - or torch.xpu.device_count() > 1 - or neuron_device_count > 1 - or tpu_device_count > 1 - ), + not (torch.cuda.device_count() > 1 or torch.xpu.device_count() > 1), reason="test requires multiple hardware accelerators", )(test_case) From 028ba1ff0bf30b5b0133a539a55b35c76ad47414 Mon Sep 17 00:00:00 2001 From: JingyaHuang Date: Thu, 19 Nov 2026 18:22:23 +0000 Subject: [PATCH 10/11] Cover submesh and from_pretrained in Neuron/TPU RNG sync tests --- tests/models/testing_utils/parallelism.py | 30 ++++++++++++++--------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/models/testing_utils/parallelism.py b/tests/models/testing_utils/parallelism.py index 9902a354ecf1..a649d2ff8eb1 100644 --- a/tests/models/testing_utils/parallelism.py +++ b/tests/models/testing_utils/parallelism.py @@ -520,7 +520,7 @@ def _parallel_rng_sync_worker( dist.all_gather(gathered_noise, noise) if rank == 0: return_dict["status"] = "success" - return_dict["same_noise"] = all(torch.equal(noise, other) for other in gathered_noise) + return_dict["same_noise_as_rank_0"] = [torch.equal(noise, other) for other in gathered_noise] # Redraw from rank 0's seed, which is what a non-parallel run with that seed would draw. torch.manual_seed(rank) seeded_noise = randn_tensor(noise.shape, device=device, dtype=noise.dtype) @@ -588,9 +588,11 @@ def _check_parallel_rng_sync( f"Parallel RNG sync failed: {return_dict.get('error', 'Unknown error')}" ) # Independent replicas are outside each other's mesh, so they must keep their own seeds. - expect_same_noise = parallelism != "tensor_parallel_submesh" - assert return_dict["same_noise"] == expect_same_noise, ( - f"Ranks drew {'different' if expect_same_noise else 'the same'} inputs from the default RNG." + rank_0_group_size = 1 if parallelism == "tensor_parallel_submesh" else world_size + expected = [rank < rank_0_group_size for rank in range(world_size)] + assert return_dict["same_noise_as_rank_0"] == expected, ( + f"Ranks drawing the same input as rank 0 from the default RNG: {return_dict['same_noise_as_rank_0']}, " + f"expected {expected}." ) assert return_dict["rank_0_seed_kept"], "Rank 0 did not draw the input a non-parallel run with its seed draws." @@ -783,8 +785,10 @@ def test_tensor_parallel_tpu_inference(self, atol=1e-3, rtol=1e-3): tp_output = torch.tensor(return_dict["output"]) torch.testing.assert_close(ref_output, tp_output, atol=atol, rtol=rtol) - def test_tensor_parallel_tpu_rng_sync(self, tmp_path, atol=2e-2, rtol=2e-2): - """Enabling TP gives every rank of the TP mesh the same default RNG.""" + @pytest.mark.parametrize("loading", ["enable_parallelism", "from_pretrained"]) + @pytest.mark.parametrize("parallelism", ["tensor_parallel", "tensor_parallel_submesh"], ids=["mesh", "submesh"]) + def test_tensor_parallel_tpu_rng_sync(self, tmp_path, parallelism, loading, atol=2e-2, rtol=2e-2): + """Loading with a TP config gives every rank of the TP mesh the same default RNG, and only those ranks.""" from torch_tpu._internal.distributed.launchers.singlehost_wrapper import prepare_tpu_environment from torch_tpu._internal.utils import hardware @@ -802,8 +806,8 @@ def test_tensor_parallel_tpu_rng_sync(self, tmp_path, atol=2e-2, rtol=2e-2): self, self.get_dummy_inputs(device="cpu"), tmp_path, - "tensor_parallel", - "enable_parallelism", + parallelism, + loading, world_size=world_size, device_type="tpu", atol=atol, @@ -866,15 +870,17 @@ def test_tensor_parallel_neuron_inference(self, atol=2e-2, rtol=2e-2): # Neuron runs matmuls in bf16 internally, so compare with a bf16-level tolerance. torch.testing.assert_close(ref_output, tp_output, atol=atol, rtol=rtol) - def test_tensor_parallel_neuron_rng_sync(self, tmp_path, atol=2e-2, rtol=2e-2): - """Enabling TP gives every rank of the TP mesh the same default RNG.""" + @pytest.mark.parametrize("loading", ["enable_parallelism", "from_pretrained"]) + @pytest.mark.parametrize("parallelism", ["tensor_parallel", "tensor_parallel_submesh"], ids=["mesh", "submesh"]) + def test_tensor_parallel_neuron_rng_sync(self, tmp_path, parallelism, loading, atol=2e-2, rtol=2e-2): + """Loading with a TP config gives every rank of the TP mesh the same default RNG, and only those ranks.""" # Inputs, and so the reference, stay on CPU: touching the Neuron device here would break the spawned workers. _check_parallel_rng_sync( self, self.get_dummy_inputs(device="cpu"), tmp_path, - "tensor_parallel", - "enable_parallelism", + parallelism, + loading, world_size=self.tp_world_size, device_type="neuron", atol=atol, From 4ec91587bf5f54e9921b93a7c3910d7744d3928d Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Fri, 20 Nov 2026 05:18:23 +0000 Subject: [PATCH 11/11] Handle list-valued main inputs in the parallel RNG sync tests Co-Authored-By: Claude Fable 5.2 --- tests/models/testing_utils/parallelism.py | 55 +++++++++++++++++------ 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/tests/models/testing_utils/parallelism.py b/tests/models/testing_utils/parallelism.py index 8205f3b7d788..71df97f1a4a9 100644 --- a/tests/models/testing_utils/parallelism.py +++ b/tests/models/testing_utils/parallelism.py @@ -522,26 +522,38 @@ def _parallel_rng_sync_worker( model.eval() inputs_on_device = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()} - noise = randn_tensor( - inputs_on_device[main_input_name].shape, device=device, dtype=inputs_on_device[main_input_name].dtype - ) - inputs_on_device[main_input_name] = noise + for k, v in inputs_on_device.items(): + if isinstance(v, list): + inputs_on_device[k] = [x.to(device) if isinstance(x, torch.Tensor) else x for x in v] + + # The main input is a tensor, or a list of them for models that take one tensor per sample. + main_input = inputs_on_device[main_input_name] + is_list = isinstance(main_input, list) + noise = [ + randn_tensor(sample.shape, device=device, dtype=sample.dtype) + for sample in (main_input if is_list else [main_input]) + ] + inputs_on_device[main_input_name] = noise if is_list else noise[0] with torch.no_grad(): output = model(**inputs_on_device, return_dict=False)[0] + if isinstance(output, (list, tuple)): + # Flatten list outputs to compare predictions with different shapes. + output = torch.cat([sample.flatten() for sample in output]) if isinstance(output, DTensor): output = output.full_tensor() - gathered_noise = [torch.empty_like(noise) for _ in range(world_size)] - dist.all_gather(gathered_noise, noise) + flat_noise = torch.cat([sample.flatten() for sample in noise]) + gathered_noise = [torch.empty_like(flat_noise) for _ in range(world_size)] + dist.all_gather(gathered_noise, flat_noise) if rank == 0: return_dict["status"] = "success" - return_dict["same_noise_as_rank_0"] = [torch.equal(noise, other) for other in gathered_noise] + return_dict["same_noise_as_rank_0"] = [torch.equal(flat_noise, other) for other in gathered_noise] # Redraw from rank 0's seed, which is what a non-parallel run with that seed would draw. torch.manual_seed(rank) - seeded_noise = randn_tensor(noise.shape, device=device, dtype=noise.dtype) - return_dict["rank_0_seed_kept"] = torch.equal(noise, seeded_noise) + seeded_noise = [randn_tensor(sample.shape, device=device, dtype=sample.dtype) for sample in noise] + return_dict["rank_0_seed_kept"] = all(torch.equal(a, b) for a, b in zip(noise, seeded_noise)) # Serialise via nested list so the manager dict can transport it across processes. - return_dict["noise"] = noise.float().cpu().tolist() + return_dict["noise"] = [sample.float().cpu().tolist() for sample in noise] return_dict["output"] = output.float().cpu().tolist() except Exception as e: @@ -576,11 +588,20 @@ def _check_parallel_rng_sync( pytest.skip(f"`num_attention_heads` ({num_heads}) is not divisible by tp_degree ({world_size}).") main_input_name = tester.main_input_name - reference_device = inputs_dict[main_input_name].device + # The main input is a tensor, or a list of them for models that take one tensor per sample. + is_list = isinstance(inputs_dict[main_input_name], list) + main_input = inputs_dict[main_input_name] if is_list else [inputs_dict[main_input_name]] + reference_device = main_input[0].device model = model_class(**init_dict).eval() checkpoint_dir = str(tmp_path / "checkpoint") model.save_pretrained(checkpoint_dir) + # Move all tensors to CPU for multiprocessing + cpu_inputs = {k: v.cpu() if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()} + for k, v in cpu_inputs.items(): + if isinstance(v, list): + cpu_inputs[k] = [x.cpu() if isinstance(x, torch.Tensor) else x for x in v] + return_dict = mp.Manager().dict() mp.spawn( _parallel_rng_sync_worker, @@ -589,7 +610,7 @@ def _check_parallel_rng_sync( _find_free_port(), model_class, checkpoint_dir, - {k: v.cpu() if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()}, + cpu_inputs, main_input_name, parallelism, loading, @@ -611,9 +632,15 @@ def _check_parallel_rng_sync( ) assert return_dict["rank_0_seed_kept"], "Rank 0 did not draw the input a non-parallel run with its seed draws." - noise = torch.tensor(return_dict["noise"], dtype=inputs_dict[main_input_name].dtype, device=reference_device) + noise = [ + torch.tensor(values, dtype=sample.dtype, device=reference_device) + for values, sample in zip(return_dict["noise"], main_input) + ] + reference_inputs = {**inputs_dict, main_input_name: noise if is_list else noise[0]} with torch.no_grad(): - reference = model.to(reference_device)(**{**inputs_dict, main_input_name: noise}, return_dict=False)[0] + reference = model.to(reference_device)(**reference_inputs, return_dict=False)[0] + if isinstance(reference, (list, tuple)): + reference = torch.cat([sample.flatten() for sample in reference]) torch.testing.assert_close(reference.float().cpu(), torch.tensor(return_dict["output"]), atol=atol, rtol=rtol)