diff --git a/src/diffusers/models/transformers/transformer_cosmos3.py b/src/diffusers/models/transformers/transformer_cosmos3.py index 57dcb8b68bfd..89a169d96712 100644 --- a/src/diffusers/models/transformers/transformer_cosmos3.py +++ b/src/diffusers/models/transformers/transformer_cosmos3.py @@ -47,6 +47,45 @@ class Cosmos3OmniTransformerOutput(BaseOutput): action: list[torch.Tensor] | None = None +class Cosmos3KVLayerCache: + """Per-layer KV cache for the text (understanding) prefix. + + Stores the und keys (post-RoPE, as read by the generation pathway) and values extracted during the first denoising + step. Tensor format: ``(und_len, num_key_value_heads, head_dim)``. + """ + + def __init__(self): + self.k: torch.Tensor | None = None + self.v: torch.Tensor | None = None + + def store(self, k: torch.Tensor, v: torch.Tensor): + self.k = k + self.v = v + + def get(self) -> tuple[torch.Tensor, torch.Tensor]: + if self.k is None: + raise RuntimeError("KV cache has not been populated yet.") + return self.k, self.v + + def clear(self): + self.k = None + self.v = None + + +class Cosmos3KVCache: + """Container for all decoder layers' text-prefix KV caches.""" + + def __init__(self, num_layers: int): + self.layer_caches = [Cosmos3KVLayerCache() for _ in range(num_layers)] + + def get_layer(self, layer_idx: int) -> Cosmos3KVLayerCache: + return self.layer_caches[layer_idx] + + def clear(self): + for cache in self.layer_caches: + cache.clear() + + class Cosmos3AttnProcessor: """Dual-pathway attention processor for Cosmos3. @@ -60,48 +99,55 @@ class Cosmos3AttnProcessor: def __call__( self, attn: "Cosmos3PackedMoTAttention", - und_seq: torch.Tensor, + und_seq: torch.Tensor | None, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - ) -> tuple[torch.Tensor, torch.Tensor]: - # Per-pathway projections - q_und = attn.to_q(und_seq).view(-1, attn.num_attention_heads, attn.head_dim) - k_und = attn.to_k(und_seq).view(-1, attn.num_key_value_heads, attn.head_dim) - v_und = attn.to_v(und_seq).view(-1, attn.num_key_value_heads, attn.head_dim) + layer_cache: Cosmos3KVLayerCache | None = None, + kv_cache_mode: str | None = None, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + cos_und, sin_und, cos_gen, sin_gen = rotary_emb + + # Causal pathway (understanding): und tokens self-attend with causal masking and never read gen tokens, so + # the und keys/values used by the gen pathway only depend on the text. "extract" stores them in + # `layer_cache`; "cached" reads them from it and skips the und pathway. + und_out = None + if kv_cache_mode == "cached": + k_und_for_gen, v_und = layer_cache.get() + else: + q_und = attn.to_q(und_seq).view(-1, attn.num_attention_heads, attn.head_dim) + k_und = attn.to_k(und_seq).view(-1, attn.num_key_value_heads, attn.head_dim) + v_und = attn.to_v(und_seq).view(-1, attn.num_key_value_heads, attn.head_dim) + q_und = attn.norm_q(q_und) + k_und = attn.norm_k(k_und) + k_und_for_gen = attn.k_norm_und_for_gen(k_und) if attn.k_norm_und_for_gen is not None else k_und + cos_und = cos_und.unsqueeze(1) + sin_und = sin_und.unsqueeze(1) + q_und = q_und * cos_und + _rotate_half(q_und) * sin_und + k_und = k_und * cos_und + _rotate_half(k_und) * sin_und + k_und_for_gen = k_und_for_gen * cos_und + _rotate_half(k_und_for_gen) * sin_und + causal_out = dispatch_attention_fn( + q_und.unsqueeze(0), + k_und.unsqueeze(0), + v_und.unsqueeze(0), + is_causal=True, + enable_gqa=True, + backend=self._attention_backend, + parallel_config=self._parallel_config, + ) + und_out = attn.to_out(causal_out.squeeze(0).flatten(-2, -1)) + if kv_cache_mode == "extract": + layer_cache.store(k_und_for_gen, v_und) + q_gen = attn.add_q_proj(gen_seq).view(-1, attn.num_attention_heads, attn.head_dim) k_gen = attn.add_k_proj(gen_seq).view(-1, attn.num_key_value_heads, attn.head_dim) v_gen = attn.add_v_proj(gen_seq).view(-1, attn.num_key_value_heads, attn.head_dim) - - q_und = attn.norm_q(q_und) - k_und = attn.norm_k(k_und) - k_und_for_gen = attn.k_norm_und_for_gen(k_und) if attn.k_norm_und_for_gen is not None else k_und q_gen = attn.norm_added_q(q_gen) k_gen = attn.norm_added_k(k_gen) - - # Apply rotary position embeddings per pathway - cos_und, sin_und, cos_gen, sin_gen = rotary_emb - cos_und = cos_und.unsqueeze(1) - sin_und = sin_und.unsqueeze(1) - q_und = q_und * cos_und + _rotate_half(q_und) * sin_und - k_und = k_und * cos_und + _rotate_half(k_und) * sin_und - k_und_for_gen = k_und_for_gen * cos_und + _rotate_half(k_und_for_gen) * sin_und cos_gen = cos_gen.unsqueeze(1) sin_gen = sin_gen.unsqueeze(1) q_gen = q_gen * cos_gen + _rotate_half(q_gen) * sin_gen k_gen = k_gen * cos_gen + _rotate_half(k_gen) * sin_gen - # Causal pathway (understanding): und tokens self-attend with causal masking. - causal_out = dispatch_attention_fn( - q_und.unsqueeze(0), - k_und.unsqueeze(0), - v_und.unsqueeze(0), - is_causal=True, - enable_gqa=True, - backend=self._attention_backend, - parallel_config=self._parallel_config, - ) - causal_out = causal_out.squeeze(0).flatten(-2, -1) - # Full pathway (generation): gen tokens cross-attend to all (und + gen) keys/values. all_k = torch.cat([k_und_for_gen, k_gen], dim=0) all_v = torch.cat([v_und, v_gen], dim=0) @@ -115,9 +161,6 @@ def __call__( parallel_config=self._parallel_config, ) full_out = full_out.squeeze(0).flatten(-2, -1) - - # Per-pathway output projection - und_out = attn.to_out(causal_out) gen_out = attn.to_add_out(full_out) return und_out, gen_out @@ -294,11 +337,15 @@ def __init__( def forward( self, - und_seq: torch.Tensor, + und_seq: torch.Tensor | None, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - ) -> tuple[torch.Tensor, torch.Tensor]: - return self.processor(self, und_seq, gen_seq, rotary_emb) + layer_cache: Cosmos3KVLayerCache | None = None, + kv_cache_mode: str | None = None, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + if kv_cache_mode is None: + return self.processor(self, und_seq, gen_seq, rotary_emb) + return self.processor(self, und_seq, gen_seq, rotary_emb, layer_cache=layer_cache, kv_cache_mode=kv_cache_mode) class Cosmos3VLTextMoTDecoderLayer(nn.Module): @@ -354,20 +401,26 @@ def __init__( def forward( self, - und_seq: torch.Tensor, + und_seq: torch.Tensor | None, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - ) -> tuple[torch.Tensor, torch.Tensor]: - und_norm = self.input_layernorm(und_seq) + layer_cache: Cosmos3KVLayerCache | None = None, + kv_cache_mode: str | None = None, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + # `und_seq` is None in "cached" mode: the und keys/values come from `layer_cache` (see `Cosmos3AttnProcessor`). + und_norm = self.input_layernorm(und_seq) if und_seq is not None else None gen_norm = self.input_layernorm_moe_gen(gen_seq) - und_attn_out, gen_attn_out = self.self_attn(und_norm, gen_norm, rotary_emb) - residual_und = und_seq + und_attn_out + und_attn_out, gen_attn_out = self.self_attn( + und_norm, gen_norm, rotary_emb, layer_cache=layer_cache, kv_cache_mode=kv_cache_mode + ) residual_gen = gen_seq + gen_attn_out - - mlp_out_und = self.mlp(self.post_attention_layernorm(residual_und)) mlp_out_gen = self.mlp_moe_gen(self.post_attention_layernorm_moe_gen(residual_gen)) + if und_seq is None: + return None, residual_gen + mlp_out_gen + residual_und = und_seq + und_attn_out + mlp_out_und = self.mlp(self.post_attention_layernorm(residual_und)) return residual_und + mlp_out_und, residual_gen + mlp_out_gen @@ -390,6 +443,15 @@ class Cosmos3OmniTransformer(ModelMixin, ConfigMixin, PeftAdapterMixin, Attentio # expressed as diffusers' declarative `_cp_plan`, so CP lives outside the model. _cp_shard_fn = None _cp_gather_fn = None + _skip_keys = ["kv_cache"] + # Hooks that only move or cast weights. Any other diffusers hook (cache, layer skip, guidance, context parallel) + # may skip, replace or reshard blocks, which `kv_cache` does not support. + _kv_cache_compatible_hooks = { + "group_offloading", + "lazy_prefetch_group_offloading", + "layerwise_casting", + "peft_autocast_disable", + } # `dtype` is injected into init_dict by ModelMixin.from_pretrained (configuration_utils.py:289), # so __init__ must accept it. Excluding it here keeps save_pretrained from writing it into # config.json — the value is a load-time runtime hint, not part of the model architecture. @@ -650,21 +712,37 @@ def _unpack_action_latents( unpacked.append(output) return unpacked + def _has_kv_cache_incompatible_hooks(self) -> bool: + # Also catches hooks added with `apply_*` functions, which do not set `_cache_config`. + for module in self.modules(): + registry = getattr(module, "_diffusers_hook", None) + if registry is not None and set(registry.hooks) - self._kv_cache_compatible_hooks: + return True + return False + def _run_decoder_stack( self, und_seq: torch.Tensor, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - ) -> tuple[torch.Tensor, torch.Tensor]: - for decoder_layer in self.layers: + kv_cache: Cosmos3KVCache | None = None, + kv_cache_mode: str | None = None, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + # In "cached" mode the und pathway is skipped: every layer reads the und keys/values from `kv_cache`. + if kv_cache_mode == "cached": + und_seq = None + for index_block, decoder_layer in enumerate(self.layers): + layer_cache = kv_cache.get_layer(index_block) if kv_cache is not None else None if torch.is_grad_enabled() and self.gradient_checkpointing: und_seq, gen_seq = self._gradient_checkpointing_func( - decoder_layer.__call__, und_seq, gen_seq, rotary_emb + decoder_layer.__call__, und_seq, gen_seq, rotary_emb, layer_cache, kv_cache_mode ) else: - und_seq, gen_seq = decoder_layer(und_seq, gen_seq, rotary_emb) + und_seq, gen_seq = decoder_layer( + und_seq, gen_seq, rotary_emb, layer_cache=layer_cache, kv_cache_mode=kv_cache_mode + ) - return self.norm(und_seq), self.norm_moe_gen(gen_seq) + return self.norm(und_seq) if und_seq is not None else None, self.norm_moe_gen(gen_seq) # ------------------------------------------------------------------------- # forward: full per-step pass — encode text/vision/sound/action → run layers → @@ -697,6 +775,8 @@ def forward( action_timesteps: torch.Tensor | None = None, action_noisy_frame_indexes: list[torch.Tensor] | None = None, action_domain_ids: list[torch.Tensor] | None = None, + kv_cache: Cosmos3KVCache | None = None, + kv_cache_mode: str | None = None, return_dict: bool = True, ) -> ( Cosmos3OmniTransformerOutput | tuple[list[torch.Tensor], list[torch.Tensor] | None, list[torch.Tensor] | None] @@ -728,12 +808,29 @@ def forward( action_timesteps: Optional per-token diffusion timesteps for action tokens. action_noisy_frame_indexes: Optional noisy frame indices per action item. action_domain_ids: Optional per-item domain IDs selecting the action head weights. + kv_cache: Optional ``Cosmos3KVCache`` holding the text-prefix keys/values. Pass together with + ``kv_cache_mode``. Valid because the und tokens never attend to the generation tokens, so their + keys/values only depend on ``input_ids`` and ``position_ids``. + kv_cache_mode: ``"extract"`` (run the und pathway and store its keys/values in ``kv_cache``) or + ``"cached"`` (skip the und pathway and read them from ``kv_cache``). Use one cache per text input, + e.g. one per CFG branch. return_dict: Whether to return a [`Cosmos3OmniTransformerOutput`] instead of a tuple. Returns: A [`Cosmos3OmniTransformerOutput`] or a tuple of per-modality prediction lists. Optional modalities return ``None`` when their inputs are omitted. """ + if kv_cache is not None and kv_cache_mode not in ("extract", "cached"): + raise ValueError( + f"kv_cache_mode must be 'extract' or 'cached' when kv_cache is provided, got {kv_cache_mode!r}." + ) + if kv_cache is None and kv_cache_mode is not None: + raise ValueError(f"kv_cache_mode is {kv_cache_mode!r} but no kv_cache was passed to hold the prefix.") + if kv_cache is not None and self._cp_shard_fn is not None: + raise ValueError("kv_cache is not supported with context parallelism.") + if kv_cache is not None and self._has_kv_cache_incompatible_hooks(): + raise ValueError("kv_cache cannot be combined with cache, layer-skip or context-parallel hooks.") + has_sound = sound_tokens is not None and sound_sequence_indexes is not None has_action = action_tokens is not None and action_sequence_indexes is not None @@ -818,7 +915,14 @@ def forward( if self._cp_shard_fn is not None: und_seq, gen_seq, rotary_emb = self._cp_shard_fn(und_seq, gen_seq, rotary_emb) - und_out, gen_out = self._run_decoder_stack(und_seq, gen_seq, rotary_emb) + # Cache hooks (e.g. SeaCache) replace `_run_decoder_stack` with a 3-argument version, so the KV-cache + # arguments are only passed when a cache is used (rejected above together with such hooks). + if kv_cache is None: + und_out, gen_out = self._run_decoder_stack(und_seq, gen_seq, rotary_emb) + else: + und_out, gen_out = self._run_decoder_stack( + und_seq, gen_seq, rotary_emb, kv_cache=kv_cache, kv_cache_mode=kv_cache_mode + ) # Optional context-parallelism gather seam: re-gather the full per-pathway # sequence on every rank (and drop the padding) before the global-index decode @@ -827,6 +931,10 @@ def forward( if self._cp_gather_fn is not None: und_out, gen_out = self._cp_gather_fn(und_out, gen_out) + # The und pathway was skipped ("cached" mode); predictions are only read at gen positions. + if und_out is None: + und_out = gen_out.new_zeros(und_len, gen_out.shape[-1]) + last_hidden_state = torch.cat([und_out, gen_out], dim=0) # Decode vision predictions from the joint hidden state. diff --git a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py index 5891e0b6651f..b85ea4287a04 100644 --- a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py +++ b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py @@ -30,6 +30,8 @@ from ...models.autoencoders.autoencoder_kl_wan import AutoencoderKLWan from ...models.modeling_utils import get_parameter_device from ...models.transformers.transformer_cosmos3 import ( + Cosmos3AttnProcessor, + Cosmos3KVCache, Cosmos3OmniTransformer, ) from ...schedulers import UniPCMultistepScheduler @@ -1350,6 +1352,7 @@ def __call__( mixed_precision_first_steps: int | None = None, mixed_precision_last_steps: int | None = None, mixed_precision_reasoner_policy: str | None = None, + use_kv_cache: bool = True, ) -> Cosmos3OmniPipelineOutput | tuple: r""" Run the Cosmos 3 omni pipeline end-to-end: encode the (optional) conditioning image/video, denoise vision and @@ -1455,6 +1458,11 @@ def __call__( Optional trailing W8A16 step count. mixed_precision_reasoner_policy (`str`, *optional*): Optional reasoner path: `"high_precision"` (W8A16) or `"base_precision"` (native W8A8). + use_kv_cache (`bool`, *optional*, defaults to `True`): + Cache the text-prefix keys and values of each CFG branch after the first step and skip the text + pathway on later steps. Valid because the text tokens never attend to the video, sound or action + tokens, so their keys and values are the same on every step. Not used with mixed precision, cache + or layer-skip hooks, context parallelism or custom attention processors. Returns: [`Cosmos3OmniPipelineOutput`] or `tuple`: @@ -1715,6 +1723,19 @@ def __call__( ) self._mixed_precision_config = mixed_precision self._mixed_precision_trace = [] + # The text (und) keys and values are the same on every step, so the first step extracts them and later steps + # only run the generation pathway. Not with step-dependent precision, cache / layer-skip hooks, context + # parallelism or custom attention processors. + cache_enabled = ( + use_kv_cache + and not mixed_precision.enabled + and not self.transformer._has_kv_cache_incompatible_hooks() + and self.transformer._cp_shard_fn is None + and all(type(p) is Cosmos3AttnProcessor for p in self.transformer.attn_processors.values()) + ) + num_layers = len(self.transformer.layers) + cond_kv_cache = Cosmos3KVCache(num_layers) if cache_enabled else None + uncond_kv_cache = Cosmos3KVCache(num_layers) if cache_enabled and self.do_classifier_free_guidance else None try: with self.progress_bar(total=num_inference_steps) as progress_bar: for i, t in enumerate(timesteps): @@ -1729,6 +1750,7 @@ def __call__( continue self._current_timestep = t + kv_cache_mode = None if cond_kv_cache is None else ("extract" if i == 0 else "cached") sigma = float(self.scheduler.sigmas[i]) timestep = t.item() @@ -1778,6 +1800,8 @@ def __call__( action_timesteps=action_timesteps, action_noisy_frame_indexes=cond_packed_static.get("action_noisy_frame_indexes"), action_domain_ids=[action_domain_id] if action_domain_id is not None else None, + kv_cache=cond_kv_cache, + kv_cache_mode=kv_cache_mode, return_dict=False, ) cond_v_vision, cond_v_sound, cond_v_action = self._mask_velocity_predictions( @@ -1821,6 +1845,8 @@ def __call__( action_timesteps=action_timesteps, action_noisy_frame_indexes=uncond_packed_static.get("action_noisy_frame_indexes"), action_domain_ids=[action_domain_id] if action_domain_id is not None else None, + kv_cache=uncond_kv_cache, + kv_cache_mode=kv_cache_mode if uncond_kv_cache is not None else None, return_dict=False, ) uncond_v_vision, uncond_v_sound, uncond_v_action = self._mask_velocity_predictions( @@ -1893,6 +1919,12 @@ def __call__( finally: reset_cosmos3_mixed_precision(self.transformer, mixed_precision) + # Clean up KV cache + if cond_kv_cache is not None: + cond_kv_cache.clear() + if uncond_kv_cache is not None: + uncond_kv_cache.clear() + self._current_timestep = None # 8. Postprocess + decode diff --git a/tests/models/transformers/test_models_transformer_cosmos3.py b/tests/models/transformers/test_models_transformer_cosmos3.py index ab1fbf7229e5..78a055e27c9a 100644 --- a/tests/models/transformers/test_models_transformer_cosmos3.py +++ b/tests/models/transformers/test_models_transformer_cosmos3.py @@ -22,6 +22,7 @@ from diffusers.hooks.sea_cache import _SEA_CACHE_ROOT_HOOK from diffusers.models.cache_utils import CacheMixin from diffusers.models.transformers.transformer_cosmos3 import ( + Cosmos3KVCache, Cosmos3NemotronRMSNorm, Cosmos3OmniTransformerOutput, Cosmos3PackedMoTAttention, @@ -676,6 +677,94 @@ def test_cosmos3_nemotron_rms_norm_multiplies_in_float32(self): torch.testing.assert_close(norm(hidden_states), expected, rtol=0, atol=0) +class TestCosmos3OmniTransformerTextKVReuse(Cosmos3OmniTransformerTesterConfig): + """`kv_cache` / `kv_cache_mode` reuse the text-prefix keys/values across denoising steps.""" + + num_steps = 3 + + def get_text_inputs(self, und_len: int, seed: int) -> dict: + # One text prefix per CFG branch; the same tensors are passed on every step, as in the pipeline. + generator = torch.Generator("cpu").manual_seed(seed) + sequence_length = und_len + 4 + return { + "input_ids": torch.randint(0, 32, (und_len,), generator=generator).to(torch_device), + "text_indexes": torch.arange(und_len, device=torch_device), + "position_ids": torch.randint(0, 16, (3, sequence_length), generator=generator).to(torch_device), + "und_len": und_len, + "sequence_length": sequence_length, + "vision_token_shapes": [(1, 2, 2)], + "vision_sequence_indexes": torch.arange(und_len, sequence_length, device=torch_device), + "vision_mse_loss_indexes": torch.arange(und_len, sequence_length, device=torch_device), + "vision_noisy_frame_indexes": [torch.tensor([0], device=torch_device)], + } + + def run_steps(self, model, branches, use_kv_cache): + # One cache per CFG branch, as in the pipeline: "extract" on the first step, "cached" afterwards. + kv_caches = [Cosmos3KVCache(len(model.layers)) if use_kv_cache else None for _ in branches] + outputs = [] + for step in range(self.num_steps): + kv_cache_mode = ("extract" if step == 0 else "cached") if use_kv_cache else None + for branch_index, (text_inputs, kv_cache) in enumerate(zip(branches, kv_caches)): + generator = torch.Generator("cpu").manual_seed(10 * step + branch_index) + vision_tokens = randn_tensor((1, 2, 1, 2, 2), generator=generator, device=torch_device) + with torch.no_grad(): + output = model( + **text_inputs, + vision_tokens=[vision_tokens], + vision_timesteps=torch.full((4,), 1.0 - 0.3 * step, device=torch_device), + kv_cache=kv_cache, + kv_cache_mode=kv_cache_mode, + ) + outputs.append(output.sample[0]) + return outputs + + def get_model_and_branches(self, use_und_k_norm_for_gen, guidance): + torch.manual_seed(0) + model = self.model_class(**self.get_init_dict(), use_und_k_norm_for_gen=use_und_k_norm_for_gen) + model = model.to(torch_device).eval() + branches = [self.get_text_inputs(und_len=3, seed=1)] + if guidance: + branches.append(self.get_text_inputs(und_len=5, seed=2)) + return model, branches + + @pytest.mark.parametrize("guidance", [False, True], ids=["no_cfg", "cfg"]) + @pytest.mark.parametrize("use_und_k_norm_for_gen", [False, True]) + def test_text_kv_reuse_matches_full_pass(self, use_und_k_norm_for_gen, guidance): + model, branches = self.get_model_and_branches(use_und_k_norm_for_gen, guidance) + + recomputed = self.run_steps(model, branches, use_kv_cache=False) + reused = self.run_steps(model, branches, use_kv_cache=True) + + assert len(reused) == len(recomputed) == self.num_steps * len(branches) + for reused_output, recomputed_output in zip(reused, recomputed): + assert torch.equal(reused_output, recomputed_output) + + @pytest.mark.parametrize("guidance", [False, True], ids=["no_cfg", "cfg"]) + def test_text_kv_reuse_runs_text_pathway_once_per_branch(self, guidance): + model, branches = self.get_model_and_branches(use_und_k_norm_for_gen=True, guidance=guidance) + text_mlp_calls = 0 + + def count_text_mlp_calls(_module, _args, _output): + nonlocal text_mlp_calls + text_mlp_calls += 1 + + for layer in model.layers: + layer.mlp.register_forward_hook(count_text_mlp_calls) + + self.run_steps(model, branches, use_kv_cache=False) + assert text_mlp_calls == len(model.layers) * self.num_steps * len(branches) + + text_mlp_calls = 0 + self.run_steps(model, branches, use_kv_cache=True) + assert text_mlp_calls == len(model.layers) * len(branches) + + def test_kv_cache_mode_requires_kv_cache(self): + model, branches = self.get_model_and_branches(use_und_k_norm_for_gen=True, guidance=False) + vision_tokens = randn_tensor((1, 2, 1, 2, 2), device=torch_device) + with pytest.raises(ValueError, match="no kv_cache"): + model(**branches[0], vision_tokens=[vision_tokens], vision_timesteps=torch.ones(4), kv_cache_mode="cached") + + class TestCosmos3OmniTransformerMemory(Cosmos3OmniTransformerTesterConfig, MemoryTesterMixin): @pytest.mark.skip("The transformer returns one tensor list per generated modality.") def test_layerwise_casting_training(self): diff --git a/tests/pipelines/cosmos/test_cosmos3.py b/tests/pipelines/cosmos/test_cosmos3.py index 5660698405c3..9ea9bdf00626 100644 --- a/tests/pipelines/cosmos/test_cosmos3.py +++ b/tests/pipelines/cosmos/test_cosmos3.py @@ -25,8 +25,11 @@ AutoencoderKLWan, Cosmos3OmniPipeline, Cosmos3OmniTransformer, + SeaCacheConfig, UniPCMultistepScheduler, + apply_sea_cache, ) +from diffusers.models.transformers.transformer_cosmos3 import Cosmos3KVCache from diffusers.pipelines.cosmos.pipeline_cosmos3_omni import _preprocess_conditioning_image from ...testing_utils import enable_full_determinism, torch_device @@ -215,6 +218,65 @@ def test_i2v_pipeline_uses_native_preprocessing(self): torch.testing.assert_close(wide_output, crop_output) + @pytest.mark.parametrize("guidance_scale", [1.0, 2.0], ids=["no_cfg", "cfg"]) + def test_text_kv_reuse_matches_recompute(self, guidance_scale): + pipeline = self.get_pipeline().to(torch_device) + pipeline.set_progress_bar_config(disable=None) + + def run(prompt, use_kv_cache): + inputs = self.get_dummy_inputs() + inputs.update( + prompt=prompt, + guidance_scale=guidance_scale, + num_inference_steps=3, + output_type="latent", + use_kv_cache=use_kv_cache, + ) + return pipeline(**inputs).video + + # Count the text-pathway MLP calls of one layer to check that the cache is actually used. + text_mlp_calls = 0 + + def count_text_mlp_calls(_module, _args, _output): + nonlocal text_mlp_calls + text_mlp_calls += 1 + + pipeline.transformer.layers[0].mlp.register_forward_hook(count_text_mlp_calls) + num_branches = 2 if guidance_scale > 1.0 else 1 + + # Two calls with different prompts: the text keys/values of the first call must not leak into the second. + prompts = ["a dog", "a cat sitting on a red mat"] + # The caches are cleared after the denoising loop, before decoding. + original_clear = Cosmos3KVCache.clear + cleared = [] + + def tracking_clear(cache): + original_clear(cache) + cleared.append(all(layer.k is None and layer.v is None for layer in cache.layer_caches)) + + with mock.patch.object(Cosmos3KVCache, "clear", tracking_clear): + reused = [run(prompt, use_kv_cache=True) for prompt in prompts] + assert cleared == [True] * (len(prompts) * num_branches) + assert text_mlp_calls == len(prompts) * num_branches + # Reference: the text pathway recomputed on every step. + text_mlp_calls = 0 + recomputed = [run(prompt, use_kv_cache=False) for prompt in prompts] + assert text_mlp_calls == len(prompts) * num_branches * 3 + + for reused_video, recomputed_video in zip(reused, recomputed): + assert torch.equal(reused_video, recomputed_video) + + def test_text_kv_reuse_off_with_cache_hooks(self): + # Hooks added with `apply_*` (not only `enable_cache`) must turn the text KV cache off instead of failing. + pipeline = self.get_pipeline().to(torch_device) + pipeline.set_progress_bar_config(disable=None) + apply_sea_cache(pipeline.transformer, SeaCacheConfig()) + assert pipeline.transformer._has_kv_cache_incompatible_hooks() + + inputs = self.get_dummy_inputs() + inputs.update(guidance_scale=2.0, num_inference_steps=3, output_type="latent") + assert pipeline(**inputs).video is not None + @pytest.mark.skip("Cosmos3 currently supports one prompt per pipeline call.") def test_inference_batch_consistent(self): pass