Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions src/diffusers/models/modeling_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1391,7 +1391,8 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike | None

parallel_config = model._resolve_parallel_config(parallel_config)
tp_config = parallel_config.tensor_parallel_config
tp_shard_specs = resolve_tp_shard_specs(model, cls._tp_plan, tp_config._mesh.size())
tp_plan = model._get_tp_plan()
tp_shard_specs = resolve_tp_shard_specs(model, tp_plan, tp_config._mesh.size())
# Each rank opens every shard file but only reads its own slices, so threading the files buys
# nothing and would have several threads calling `register_parameter` on the same modules.
if is_parallel_loading_enabled:
Expand Down Expand Up @@ -1560,7 +1561,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike | None
module = model.get_submodule(module_path) if module_path else model
module._buffers[buffer_name] = buffer.to(tp_device)

apply_tensor_parallel(model, tp_config, cls._tp_plan, weights_already_sharded=True)
apply_tensor_parallel(model, tp_config, tp_plan, weights_already_sharded=True)
else:
model.enable_parallelism(config=parallel_config)

Expand Down Expand Up @@ -1753,6 +1754,9 @@ def _resolve_parallel_config(
self._parallel_config = config
return config

def _get_tp_plan(self):
return self._tp_plan

def enable_parallelism(
self,
*,
Expand Down Expand Up @@ -1781,16 +1785,17 @@ def enable_parallelism(
if tp_config is not None:
from ..hooks.tensor_parallel import _check_tp_supported, _tp_degree, resolve_tp_shard_specs

tp_plan = self._get_tp_plan()
# Before `_resolve_parallel_config`, which records the config on the model: a model that fails these
# checks is left untouched.
_check_tp_supported(
self.__class__.__name__,
self._tp_plan,
tp_plan,
getattr(self.config, "num_attention_heads", None),
tp_config,
model=self,
)
resolve_tp_shard_specs(self, self._tp_plan, _tp_degree(tp_config))
resolve_tp_shard_specs(self, tp_plan, _tp_degree(tp_config))
logger.warning(
"Sharding an in-memory model costs full checkpoint memory on every rank. Prefer "
"`from_pretrained(..., parallel_config=...)`, which reads only each rank's shard."
Expand Down Expand Up @@ -1850,7 +1855,7 @@ def enable_parallelism(
if config.tensor_parallel_config is not None:
from ..hooks.tensor_parallel import apply_tensor_parallel

apply_tensor_parallel(self, config.tensor_parallel_config, self._tp_plan)
apply_tensor_parallel(self, config.tensor_parallel_config, tp_plan)

@classmethod
def _load_pretrained_model(
Expand Down
2 changes: 1 addition & 1 deletion tests/models/testing_utils/parallelism.py
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ def _tensor_parallel_from_pretrained_worker(
output = output.full_tensor()

if rank == 0:
specs = resolve_tp_shard_specs(model, model_class._tp_plan, world_size)
specs = resolve_tp_shard_specs(model, model._get_tp_plan(), world_size)
state_dict = model.state_dict()

# The numerics check alone wouldn't catch a planned parameter left unsharded.
Expand Down
Loading