Repository navigation
[Cosmos3] Add native tensor parallelism #15137
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
a1b567f
971bf9a
ef4beac
77ed5c4
02631f9
2e7c7c7
96a7dc7
f09416b
e06b3b4
c4fc346
fe5488a
504306b
ba9f18f
9f4be5f
9f64b3e
e041ca1
96ef545
1f1f159
47b45c3
cd5c5fc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -36,6 +36,7 @@ | |
| MemoryTesterMixin, | ||
| ModelTesterMixin, | ||
| SeaCacheTesterMixin, | ||
| TensorParallelTesterMixin, | ||
| TorchCompileTesterMixin, | ||
| TrainingTesterMixin, | ||
| ) | ||
|
|
@@ -714,3 +715,49 @@ def test_mixed_precision_training(self): | |
|
|
||
| class TestCosmos3OmniTransformerAttention(Cosmos3OmniTransformerTesterConfig, AttentionTesterMixin): | ||
| pass | ||
|
|
||
|
|
||
| class Cosmos3TensorParallelTesterConfig(Cosmos3OmniTransformerTesterConfig): | ||
| hidden_act = "relu2" | ||
|
|
||
| def get_init_dict(self): | ||
| return { | ||
| **super().get_init_dict(), | ||
| "head_dim": 8, | ||
| "hidden_size": 64, | ||
| "intermediate_size": 128, | ||
| "num_attention_heads": 8, | ||
| "num_key_value_heads": 4, | ||
| "rope_axes_dim": [2, 1, 1], | ||
| "attention_bias": True, | ||
| "hidden_act": self.hidden_act, | ||
| "qk_norm_for_text": self.hidden_act == "silu", | ||
| "use_und_k_norm_for_gen": self.hidden_act == "relu2", | ||
| } | ||
|
|
||
| def get_dummy_inputs(self, batch_size=1): | ||
| inputs = super().get_dummy_inputs(width=7) | ||
| device = inputs["input_ids"].device | ||
| und_len, gen_len = 5 * batch_size, 7 * batch_size | ||
| indexes = torch.arange(und_len, und_len + gen_len, device=device) | ||
| inputs.update( | ||
| input_ids=torch.arange(und_len, device=device) % self.get_init_dict()["vocab_size"], | ||
| text_indexes=torch.arange(und_len, device=device), | ||
| position_ids=torch.arange(und_len + gen_len, device=device).expand(3, -1), | ||
| und_len=und_len, | ||
| sequence_length=und_len + gen_len, | ||
| vision_tokens=[inputs["vision_tokens"][0] + index for index in range(batch_size)], | ||
| vision_token_shapes=inputs["vision_token_shapes"] * batch_size, | ||
| vision_sequence_indexes=indexes, | ||
| vision_mse_loss_indexes=indexes, | ||
| vision_timesteps=torch.ones(gen_len, device=device), | ||
| vision_noisy_frame_indexes=inputs["vision_noisy_frame_indexes"] * batch_size, | ||
| ) | ||
| return inputs | ||
|
|
||
|
|
||
| class TestCosmos3OmniTransformerTensorParallel(Cosmos3TensorParallelTesterConfig, TensorParallelTesterMixin): | ||
| # Run every inherited TP test for both SiLU (Super) and ReLU2 (Edge) MLP layouts. | ||
| @pytest.fixture(autouse=True, params=["silu", "relu2"]) | ||
| def activation(self, request): | ||
| self.hidden_act = request.param | ||
|
Comment on lines
+761
to
+763
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why do we need this fixture?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This runs the inherited TP tests against both Cosmos MLP layouts: SiLU has gate projections, while ReLU2 does not. The fixture selects the configuration without changing the shared test mixin |
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
seems safer?