diff --git a/docs/source/en/api/pipelines/cosmos3.md b/docs/source/en/api/pipelines/cosmos3.md index 4e051c3f98a5..f0efad41f1fe 100644 --- a/docs/source/en/api/pipelines/cosmos3.md +++ b/docs/source/en/api/pipelines/cosmos3.md @@ -764,7 +764,39 @@ torchrun --nproc_per_node=4 examples/cosmos3/inference_cosmos3.py \ ### Fitting large models with tensor parallelism -CP shards *activations* but replicates every weight on every rank, so it does not reduce a model's weight footprint — a model that doesn't fit on one GPU still won't fit under CP alone. To shard the **weights**, `enable_cosmos3_tensor_parallel(transformer, tp_mesh)` applies Megatron-style tensor parallelism on a second, orthogonal mesh axis: +CP shards *activations* but replicates every weight on every rank, so it does not reduce a model's weight footprint — a model that doesn't fit on one GPU still won't fit under CP alone. + +For TP-only inference, follow the [tensor parallelism guide](../../optimization/tensor_parallelism) +for distributed setup, launching, seeding, and rank-zero output handling. After initializing the process group +and selecting each rank's CUDA device, load the modular pipeline as follows: + +```python +import torch +from diffusers import Cosmos3OmniModularPipeline, Cosmos3OmniTransformer, TensorParallelConfig + +model_id = "nvidia/Cosmos3-Super" +device = torch.device("cuda", torch.cuda.current_device()) +transformer = Cosmos3OmniTransformer.from_pretrained( + model_id, + subfolder="transformer", + dtype=torch.bfloat16, + parallel_config=TensorParallelConfig(tp_degree=4), +) +pipe = Cosmos3OmniModularPipeline.from_pretrained(model_id) +pipe.update_components(transformer=transformer) +pipe.load_components(dtype=torch.bfloat16) +pipe.enable_safety_checker() +pipe.to(device) +``` + +Continue with the [existing modular workflow](#modular-examples-for-all-existing-workflows)'s scheduler, +prompts, and generation settings. To use Nano, set `model_id = "nvidia/Cosmos3-Nano"`. + +Nano and Super support TP degrees 1, 2, 4, and 8; the degree must divide the query and KV head counts +and the MLP intermediate size. + +To combine TP with CP, the existing example helper +`enable_cosmos3_tensor_parallel(transformer, tp_mesh)` shards the weights on a second, orthogonal mesh axis: - The attention and MLP projections are column/row sharded across the TP group (`to_q/to_k/to_v` + `add_q/k/v` and the MLPs' `gate/up` are column-parallel; `to_out/to_add_out` and the MLPs' `down` are row-parallel with an all-reduce). Each rank ends up owning `query_heads / tp` query heads and `kv_heads / tp` KV heads. - TP composes with CP on a 2-D `(tp, cp)` device mesh: TP splits heads/weights persistently, CP shards the sequence on top. The constraints are `tp` divides the KV heads (8), and `tp * cp` divides the query heads (32 for `Nano`, 64 for `Super`). diff --git a/src/diffusers/hooks/tensor_parallel.py b/src/diffusers/hooks/tensor_parallel.py index 8e98a507ee46..06a75b71556c 100644 --- a/src/diffusers/hooks/tensor_parallel.py +++ b/src/diffusers/hooks/tensor_parallel.py @@ -424,6 +424,7 @@ def _check_tp_supported( num_heads: "int | None", tp_config, *, + num_key_value_heads: "int | None" = None, model: "torch.nn.Module | None" = None, device_map=None, hf_quantizer=None, @@ -448,6 +449,10 @@ def _check_tp_supported( tp_degree = _tp_degree(tp_config) if num_heads is not None and num_heads % tp_degree != 0: raise ValueError(f"`tp_degree` ({tp_degree}) must divide the number of attention heads ({num_heads}).") + if num_key_value_heads is not None and num_key_value_heads % tp_degree != 0: + raise ValueError( + f"`tp_degree` ({tp_degree}) must divide the number of key/value heads ({num_key_value_heads})." + ) if device_map is not None: raise ValueError( diff --git a/src/diffusers/models/modeling_utils.py b/src/diffusers/models/modeling_utils.py index 2f45d20adc18..2b2d78f7c23d 100644 --- a/src/diffusers/models/modeling_utils.py +++ b/src/diffusers/models/modeling_utils.py @@ -1235,6 +1235,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: str | os.PathLike | None cls._tp_plan, config.get("num_attention_heads"), tp_config, + num_key_value_heads=config.get("num_key_value_heads"), device_map=device_map, low_cpu_mem_usage=low_cpu_mem_usage, use_flashpack=use_flashpack, @@ -1756,6 +1757,7 @@ def enable_parallelism( self._tp_plan, getattr(self.config, "num_attention_heads", None), tp_config, + num_key_value_heads=getattr(self.config, "num_key_value_heads", None), model=self, ) resolve_tp_shard_specs(self, self._tp_plan, _tp_degree(tp_config)) diff --git a/src/diffusers/models/transformers/transformer_cosmos3.py b/src/diffusers/models/transformers/transformer_cosmos3.py index 57dcb8b68bfd..1c9d87b784f9 100644 --- a/src/diffusers/models/transformers/transformer_cosmos3.py +++ b/src/diffusers/models/transformers/transformer_cosmos3.py @@ -21,6 +21,7 @@ from ...configuration_utils import ConfigMixin, register_to_config from ...loaders import PeftAdapterMixin from ...utils import BaseOutput +from .._modeling_parallel import Conditional from ..attention import AttentionMixin, AttentionModuleMixin from ..attention_dispatch import dispatch_attention_fn from ..cache_utils import CacheMixin @@ -64,13 +65,13 @@ def __call__( gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], ) -> tuple[torch.Tensor, torch.Tensor]: - # Per-pathway projections - q_und = attn.to_q(und_seq).view(-1, attn.num_attention_heads, attn.head_dim) - k_und = attn.to_k(und_seq).view(-1, attn.num_key_value_heads, attn.head_dim) - v_und = attn.to_v(und_seq).view(-1, attn.num_key_value_heads, attn.head_dim) - q_gen = attn.add_q_proj(gen_seq).view(-1, attn.num_attention_heads, attn.head_dim) - k_gen = attn.add_k_proj(gen_seq).view(-1, attn.num_key_value_heads, attn.head_dim) - v_gen = attn.add_v_proj(gen_seq).view(-1, attn.num_key_value_heads, attn.head_dim) + # Infer local head counts from the projection widths after tensor parallel sharding. + q_und = attn.to_q(und_seq).unflatten(-1, (-1, attn.head_dim)) + k_und = attn.to_k(und_seq).unflatten(-1, (-1, attn.head_dim)) + v_und = attn.to_v(und_seq).unflatten(-1, (-1, attn.head_dim)) + q_gen = attn.add_q_proj(gen_seq).unflatten(-1, (-1, attn.head_dim)) + k_gen = attn.add_k_proj(gen_seq).unflatten(-1, (-1, attn.head_dim)) + v_gen = attn.add_v_proj(gen_seq).unflatten(-1, (-1, attn.head_dim)) q_und = attn.norm_q(q_und) k_und = attn.norm_k(k_und) @@ -377,6 +378,16 @@ class Cosmos3OmniTransformer(ModelMixin, ConfigMixin, PeftAdapterMixin, Attentio _repeated_blocks = ["Cosmos3VLTextMoTDecoderLayer"] _skip_layerwise_casting_patterns = ["embed_tokens", "time_embedder", "norm"] _keep_in_fp32_modules = ["time_embedder"] + _tp_plan = { + **{ + f"layers.*.self_attn.{name}": "colwise" + for name in ("to_q", "to_k", "to_v", "add_q_proj", "add_k_proj", "add_v_proj") + }, + **{f"layers.*.self_attn.{name}": "rowwise" for name in ("to_out", "to_add_out")}, + **{f"layers.*.{mlp}.up_proj": "colwise" for mlp in ("mlp", "mlp_moe_gen")}, + **{f"layers.*.{mlp}.gate_proj": Conditional("colwise") for mlp in ("mlp", "mlp_moe_gen")}, + **{f"layers.*.{mlp}.down_proj": "rowwise" for mlp in ("mlp", "mlp_moe_gen")}, + } # These scale tensors are exported for vLLM-Omni/SGLang and are unused # by Diffusers. ModelOpt restores its quantizer state separately. _keys_to_ignore_on_load_unexpected = [r"\.(?:input_scale|weight_scale)$"] diff --git a/tests/models/testing_utils/parallelism.py b/tests/models/testing_utils/parallelism.py index a4da999cb350..d73de18b99f5 100644 --- a/tests/models/testing_utils/parallelism.py +++ b/tests/models/testing_utils/parallelism.py @@ -332,6 +332,8 @@ def _tensor_parallel_worker( to the unsharded model up to floating-point reduction order). """ try: + from torch.distributed.tensor import DTensor + os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = str(master_port) os.environ["RANK"] = str(rank) @@ -355,12 +357,20 @@ def _tensor_parallel_worker( model.eval() inputs_on_device = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()} + for k, v in inputs_on_device.items(): + if isinstance(v, list): + inputs_on_device[k] = [x.to(device) if isinstance(x, torch.Tensor) else x for x in v] # Shard the model across all ranks; the device mesh is built from `tp_degree` on the active accelerator. model.enable_parallelism(config=TensorParallelConfig(tp_degree=world_size)) with torch.no_grad(): output = model(**inputs_on_device, return_dict=False)[0] + if isinstance(output, (list, tuple)): + # Flatten list outputs to compare predictions with different shapes. + output = torch.cat([sample.flatten() for sample in output]) + if isinstance(output, DTensor): + output = output.full_tensor() if rank == 0: return_dict["status"] = "success" @@ -406,8 +416,13 @@ def _tensor_parallel_from_pretrained_worker( device = torch.device(f"{torch_device}:{rank}") inputs_on_device = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()} + for k, v in inputs_on_device.items(): + if isinstance(v, list): + inputs_on_device[k] = [x.to(device) if isinstance(x, torch.Tensor) else x for x in v] with torch.no_grad(): output = model(**inputs_on_device, return_dict=False)[0] + if isinstance(output, (list, tuple)): + output = torch.cat([sample.flatten() for sample in output]) if isinstance(output, DTensor): output = output.full_tensor() @@ -469,10 +484,16 @@ def test_tensor_parallel_inference(self, batch_size: int = 1): model = self.model_class(**init_dict).eval().to(torch_device) state_dict = {k: v.cpu() for k, v in model.state_dict().items()} with torch.no_grad(): - ref_output = model(**inputs_dict, return_dict=False)[0].float().cpu() + ref_output = model(**inputs_dict, return_dict=False)[0] + if isinstance(ref_output, (list, tuple)): + ref_output = torch.cat([sample.flatten() for sample in ref_output]) + ref_output = ref_output.float().cpu() # Move all tensors to CPU for multiprocessing inputs_dict = {k: v.cpu() if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()} + for k, v in inputs_dict.items(): + if isinstance(v, list): + inputs_dict[k] = [x.cpu() if isinstance(x, torch.Tensor) else x for x in v] master_port = _find_free_port() manager = mp.Manager() @@ -515,7 +536,10 @@ def _tp_checkpoint_and_reference(self, tmp_path, world_size, sharded): inputs_dict = self.get_dummy_inputs() model = self.model_class(**init_dict).eval().to(torch_device) with torch.no_grad(): - reference = model(**inputs_dict, return_dict=False)[0].float().cpu() + reference = model(**inputs_dict, return_dict=False)[0] + if isinstance(reference, (list, tuple)): + reference = torch.cat([sample.flatten() for sample in reference]) + reference = reference.float().cpu() checkpoint_dir = str(tmp_path / "checkpoint") max_shard_size = int(compute_module_persistent_sizes(model)[""] * 0.75) if sharded else "5GB" @@ -529,6 +553,9 @@ def _tp_checkpoint_and_reference(self, tmp_path, world_size, sharded): assert actual_num_shards == expected_num_shards > 1 inputs_dict = {k: v.cpu() if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()} + for k, v in inputs_dict.items(): + if isinstance(v, list): + inputs_dict[k] = [x.cpu() if isinstance(x, torch.Tensor) else x for x in v] return checkpoint_dir, inputs_dict, reference @pytest.mark.parametrize("sharded", [False, True], ids=["unsharded", "sharded"]) diff --git a/tests/models/transformers/test_models_transformer_cosmos3.py b/tests/models/transformers/test_models_transformer_cosmos3.py index ab1fbf7229e5..9227bedbaca1 100644 --- a/tests/models/transformers/test_models_transformer_cosmos3.py +++ b/tests/models/transformers/test_models_transformer_cosmos3.py @@ -36,6 +36,7 @@ MemoryTesterMixin, ModelTesterMixin, SeaCacheTesterMixin, + TensorParallelTesterMixin, TorchCompileTesterMixin, TrainingTesterMixin, ) @@ -714,3 +715,49 @@ def test_mixed_precision_training(self): class TestCosmos3OmniTransformerAttention(Cosmos3OmniTransformerTesterConfig, AttentionTesterMixin): pass + + +class Cosmos3TensorParallelTesterConfig(Cosmos3OmniTransformerTesterConfig): + hidden_act = "relu2" + + def get_init_dict(self): + return { + **super().get_init_dict(), + "head_dim": 8, + "hidden_size": 64, + "intermediate_size": 128, + "num_attention_heads": 8, + "num_key_value_heads": 4, + "rope_axes_dim": [2, 1, 1], + "attention_bias": True, + "hidden_act": self.hidden_act, + "qk_norm_for_text": self.hidden_act == "silu", + "use_und_k_norm_for_gen": self.hidden_act == "relu2", + } + + def get_dummy_inputs(self, batch_size=1): + inputs = super().get_dummy_inputs(width=7) + device = inputs["input_ids"].device + und_len, gen_len = 5 * batch_size, 7 * batch_size + indexes = torch.arange(und_len, und_len + gen_len, device=device) + inputs.update( + input_ids=torch.arange(und_len, device=device) % self.get_init_dict()["vocab_size"], + text_indexes=torch.arange(und_len, device=device), + position_ids=torch.arange(und_len + gen_len, device=device).expand(3, -1), + und_len=und_len, + sequence_length=und_len + gen_len, + vision_tokens=[inputs["vision_tokens"][0] + index for index in range(batch_size)], + vision_token_shapes=inputs["vision_token_shapes"] * batch_size, + vision_sequence_indexes=indexes, + vision_mse_loss_indexes=indexes, + vision_timesteps=torch.ones(gen_len, device=device), + vision_noisy_frame_indexes=inputs["vision_noisy_frame_indexes"] * batch_size, + ) + return inputs + + +class TestCosmos3OmniTransformerTensorParallel(Cosmos3TensorParallelTesterConfig, TensorParallelTesterMixin): + # Run every inherited TP test for both SiLU (Super) and ReLU2 (Edge) MLP layouts. + @pytest.fixture(autouse=True, params=["silu", "relu2"]) + def activation(self, request): + self.hidden_act = request.param