diff --git a/src/diffusers/models/modeling_utils.py b/src/diffusers/models/modeling_utils.py index 0f494e8158b7..915e357345f8 100644 --- a/src/diffusers/models/modeling_utils.py +++ b/src/diffusers/models/modeling_utils.py @@ -1391,7 +1391,8 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike | None parallel_config = model._resolve_parallel_config(parallel_config) tp_config = parallel_config.tensor_parallel_config - tp_shard_specs = resolve_tp_shard_specs(model, cls._tp_plan, tp_config._mesh.size()) + tp_plan = model._get_tp_plan() + tp_shard_specs = resolve_tp_shard_specs(model, tp_plan, tp_config._mesh.size()) # Each rank opens every shard file but only reads its own slices, so threading the files buys # nothing and would have several threads calling `register_parameter` on the same modules. if is_parallel_loading_enabled: @@ -1560,7 +1561,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike | None module = model.get_submodule(module_path) if module_path else model module._buffers[buffer_name] = buffer.to(tp_device) - apply_tensor_parallel(model, tp_config, cls._tp_plan, weights_already_sharded=True) + apply_tensor_parallel(model, tp_config, tp_plan, weights_already_sharded=True) else: model.enable_parallelism(config=parallel_config) @@ -1753,6 +1754,9 @@ def _resolve_parallel_config( self._parallel_config = config return config + def _get_tp_plan(self): + return self._tp_plan + def enable_parallelism( self, *, @@ -1781,16 +1785,17 @@ def enable_parallelism( if tp_config is not None: from ..hooks.tensor_parallel import _check_tp_supported, _tp_degree, resolve_tp_shard_specs + tp_plan = self._get_tp_plan() # Before `_resolve_parallel_config`, which records the config on the model: a model that fails these # checks is left untouched. _check_tp_supported( self.__class__.__name__, - self._tp_plan, + tp_plan, getattr(self.config, "num_attention_heads", None), tp_config, model=self, ) - resolve_tp_shard_specs(self, self._tp_plan, _tp_degree(tp_config)) + resolve_tp_shard_specs(self, tp_plan, _tp_degree(tp_config)) logger.warning( "Sharding an in-memory model costs full checkpoint memory on every rank. Prefer " "`from_pretrained(..., parallel_config=...)`, which reads only each rank's shard." @@ -1850,7 +1855,7 @@ def enable_parallelism( if config.tensor_parallel_config is not None: from ..hooks.tensor_parallel import apply_tensor_parallel - apply_tensor_parallel(self, config.tensor_parallel_config, self._tp_plan) + apply_tensor_parallel(self, config.tensor_parallel_config, tp_plan) @classmethod def _load_pretrained_model( diff --git a/tests/models/testing_utils/parallelism.py b/tests/models/testing_utils/parallelism.py index de19a7edd212..8bc6c0485386 100644 --- a/tests/models/testing_utils/parallelism.py +++ b/tests/models/testing_utils/parallelism.py @@ -347,7 +347,7 @@ def _tensor_parallel_from_pretrained_worker( output = output.full_tensor() if rank == 0: - specs = resolve_tp_shard_specs(model, model_class._tp_plan, world_size) + specs = resolve_tp_shard_specs(model, model._get_tp_plan(), world_size) state_dict = model.state_dict() # The numerics check alone wouldn't catch a planned parameter left unsharded.