From ff7a29269fac17cdd9ac7fee416f8eefca95ad74 Mon Sep 17 00:00:00 2001 From: "Hsu, Chan Fu" Date: Sat, 14 Nov 2026 14:28:32 +0800 Subject: [PATCH 1/5] Reuse the Cosmos3 text-prefix keys/values across denoising steps Cosmos3OmniTransformer ran the und pathway (the causal text prefix) of every decoder layer on every transformer call. The und tokens attend only to und tokens and start from embed_tokens(input_ids), and Cosmos3OmniPipeline passes the same input_ids / position_ids on every step, so every layer's und keys and values are the same on every step of one pipeline call. The gen pathway reads them only through the key/value concatenation. Store each layer's post-RoPE und keys and values on the first transformer call of each CFG branch and skip the und pathway on the later calls. The pipeline enables this for one __call__ through a private transformer attribute, cleared in the existing finally; it stays off with mixed precision, cache hooks, custom attention processors, context parallelism and gradients. Add tests that check the transformer outputs with the reused keys and values, bit for bit, against the full pass (with and without CFG, with and without the und key norm for gen), that the und pathway runs once per CFG branch, and that the pipeline output over two calls with different prompts equals the output with the reuse disabled. Bit-identical nvidia/Cosmos3-Edge pipeline output in eager, bf16, on RTX 5090 at the model card settings (480x832, 121 frames, 20 steps, guidance 6.0). Signed-off-by: Hsu, Chan Fu --- .../transformers/transformer_cosmos3.py | 135 +++++++++++------- .../pipelines/cosmos/pipeline_cosmos3_omni.py | 10 ++ .../test_models_transformer_cosmos3.py | 81 +++++++++++ tests/pipelines/cosmos/test_cosmos3.py | 25 ++++ 4 files changed, 203 insertions(+), 48 deletions(-) diff --git a/src/diffusers/models/transformers/transformer_cosmos3.py b/src/diffusers/models/transformers/transformer_cosmos3.py index 0295e1c0567b..344c2161fd0d 100644 --- a/src/diffusers/models/transformers/transformer_cosmos3.py +++ b/src/diffusers/models/transformers/transformer_cosmos3.py @@ -60,48 +60,54 @@ class Cosmos3AttnProcessor: def __call__( self, attn: "Cosmos3PackedMoTAttention", - und_seq: torch.Tensor, + und_seq: torch.Tensor | None, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - ) -> tuple[torch.Tensor, torch.Tensor]: - # Per-pathway projections - q_und = attn.to_q(und_seq).view(-1, attn.num_attention_heads, attn.head_dim) - k_und = attn.to_k(und_seq).view(-1, attn.num_key_value_heads, attn.head_dim) - v_und = attn.to_v(und_seq).view(-1, attn.num_key_value_heads, attn.head_dim) + und_kv: list[torch.Tensor] | None = None, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + cos_und, sin_und, cos_gen, sin_gen = rotary_emb + + # Causal pathway (understanding): und tokens self-attend with causal masking and never read gen tokens, so + # the und keys/values used by the gen pathway only depend on the text. When `und_kv` is given it is filled + # with them; when `und_seq` is None they are taken from `und_kv` and the und pathway is skipped. + und_out = None + if und_seq is None: + k_und_for_gen, v_und = und_kv + else: + q_und = attn.to_q(und_seq).view(-1, attn.num_attention_heads, attn.head_dim) + k_und = attn.to_k(und_seq).view(-1, attn.num_key_value_heads, attn.head_dim) + v_und = attn.to_v(und_seq).view(-1, attn.num_key_value_heads, attn.head_dim) + q_und = attn.norm_q(q_und) + k_und = attn.norm_k(k_und) + k_und_for_gen = attn.k_norm_und_for_gen(k_und) if attn.k_norm_und_for_gen is not None else k_und + cos_und = cos_und.unsqueeze(1) + sin_und = sin_und.unsqueeze(1) + q_und = q_und * cos_und + _rotate_half(q_und) * sin_und + k_und = k_und * cos_und + _rotate_half(k_und) * sin_und + k_und_for_gen = k_und_for_gen * cos_und + _rotate_half(k_und_for_gen) * sin_und + causal_out = dispatch_attention_fn( + q_und.unsqueeze(0), + k_und.unsqueeze(0), + v_und.unsqueeze(0), + is_causal=True, + enable_gqa=True, + backend=self._attention_backend, + parallel_config=self._parallel_config, + ) + und_out = attn.to_out(causal_out.squeeze(0).flatten(-2, -1)) + if und_kv is not None: + und_kv[:] = [k_und_for_gen, v_und] + q_gen = attn.add_q_proj(gen_seq).view(-1, attn.num_attention_heads, attn.head_dim) k_gen = attn.add_k_proj(gen_seq).view(-1, attn.num_key_value_heads, attn.head_dim) v_gen = attn.add_v_proj(gen_seq).view(-1, attn.num_key_value_heads, attn.head_dim) - - q_und = attn.norm_q(q_und) - k_und = attn.norm_k(k_und) - k_und_for_gen = attn.k_norm_und_for_gen(k_und) if attn.k_norm_und_for_gen is not None else k_und q_gen = attn.norm_added_q(q_gen) k_gen = attn.norm_added_k(k_gen) - - # Apply rotary position embeddings per pathway - cos_und, sin_und, cos_gen, sin_gen = rotary_emb - cos_und = cos_und.unsqueeze(1) - sin_und = sin_und.unsqueeze(1) - q_und = q_und * cos_und + _rotate_half(q_und) * sin_und - k_und = k_und * cos_und + _rotate_half(k_und) * sin_und - k_und_for_gen = k_und_for_gen * cos_und + _rotate_half(k_und_for_gen) * sin_und cos_gen = cos_gen.unsqueeze(1) sin_gen = sin_gen.unsqueeze(1) q_gen = q_gen * cos_gen + _rotate_half(q_gen) * sin_gen k_gen = k_gen * cos_gen + _rotate_half(k_gen) * sin_gen - # Causal pathway (understanding): und tokens self-attend with causal masking. - causal_out = dispatch_attention_fn( - q_und.unsqueeze(0), - k_und.unsqueeze(0), - v_und.unsqueeze(0), - is_causal=True, - enable_gqa=True, - backend=self._attention_backend, - parallel_config=self._parallel_config, - ) - causal_out = causal_out.squeeze(0).flatten(-2, -1) - # Full pathway (generation): gen tokens cross-attend to all (und + gen) keys/values. all_k = torch.cat([k_und_for_gen, k_gen], dim=0) all_v = torch.cat([v_und, v_gen], dim=0) @@ -115,9 +121,6 @@ def __call__( parallel_config=self._parallel_config, ) full_out = full_out.squeeze(0).flatten(-2, -1) - - # Per-pathway output projection - und_out = attn.to_out(causal_out) gen_out = attn.to_add_out(full_out) return und_out, gen_out @@ -294,11 +297,14 @@ def __init__( def forward( self, - und_seq: torch.Tensor, + und_seq: torch.Tensor | None, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - ) -> tuple[torch.Tensor, torch.Tensor]: - return self.processor(self, und_seq, gen_seq, rotary_emb) + und_kv: list[torch.Tensor] | None = None, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + if und_kv is None: + return self.processor(self, und_seq, gen_seq, rotary_emb) + return self.processor(self, und_seq, gen_seq, rotary_emb, und_kv=und_kv) class Cosmos3VLTextMoTDecoderLayer(nn.Module): @@ -354,20 +360,23 @@ def __init__( def forward( self, - und_seq: torch.Tensor, + und_seq: torch.Tensor | None, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - ) -> tuple[torch.Tensor, torch.Tensor]: - und_norm = self.input_layernorm(und_seq) + und_kv: list[torch.Tensor] | None = None, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + # `und_seq` is None when the und keys/values come from `und_kv` (see `Cosmos3AttnProcessor`). + und_norm = self.input_layernorm(und_seq) if und_seq is not None else None gen_norm = self.input_layernorm_moe_gen(gen_seq) - und_attn_out, gen_attn_out = self.self_attn(und_norm, gen_norm, rotary_emb) - residual_und = und_seq + und_attn_out + und_attn_out, gen_attn_out = self.self_attn(und_norm, gen_norm, rotary_emb, und_kv=und_kv) residual_gen = gen_seq + gen_attn_out - - mlp_out_und = self.mlp(self.post_attention_layernorm(residual_und)) mlp_out_gen = self.mlp_moe_gen(self.post_attention_layernorm_moe_gen(residual_gen)) + if und_seq is None: + return None, residual_gen + mlp_out_gen + residual_und = und_seq + und_attn_out + mlp_out_und = self.mlp(self.post_attention_layernorm(residual_und)) return residual_und + mlp_out_und, residual_gen + mlp_out_gen @@ -387,6 +396,11 @@ class Cosmos3OmniTransformer(ModelMixin, ConfigMixin, PeftAdapterMixin, Attentio # expressed as diffusers' declarative `_cp_plan`, so CP lives outside the model. _cp_shard_fn = None _cp_gather_fn = None + # Optional reuse of the text (und) pathway across denoising steps. The und tokens never attend to the gen tokens, + # so for the same `input_ids` / `position_ids` every layer's und keys/values are the same on every step. + # `Cosmos3OmniPipeline` sets this to an empty dict for the duration of one `__call__`; `None` disables it. + _und_kv_cache = None + _skip_keys = ["und_kv"] # `dtype` is injected into init_dict by ModelMixin.from_pretrained (configuration_utils.py:289), # so __init__ must accept it. Excluding it here keeps save_pretrained from writing it into # config.json — the value is a load-time runtime hint, not part of the model architecture. @@ -652,16 +666,22 @@ def _run_decoder_stack( und_seq: torch.Tensor, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - ) -> tuple[torch.Tensor, torch.Tensor]: - for decoder_layer in self.layers: + und_kv: list[list[torch.Tensor]] | None = None, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + # `und_kv`: one list per layer. Empty lists are filled by this pass; filled lists replace the und pathway. + if und_kv is not None and und_kv[-1]: + und_seq = None + for i, decoder_layer in enumerate(self.layers): if torch.is_grad_enabled() and self.gradient_checkpointing: und_seq, gen_seq = self._gradient_checkpointing_func( decoder_layer.__call__, und_seq, gen_seq, rotary_emb ) - else: + elif und_kv is None: und_seq, gen_seq = decoder_layer(und_seq, gen_seq, rotary_emb) + else: + und_seq, gen_seq = decoder_layer(und_seq, gen_seq, rotary_emb, und_kv=und_kv[i]) - return self.norm(und_seq), self.norm_moe_gen(gen_seq) + return self.norm(und_seq) if und_seq is not None else None, self.norm_moe_gen(gen_seq) # ------------------------------------------------------------------------- # forward: full per-step pass — encode text/vision/sound/action → run layers → @@ -815,7 +835,22 @@ def forward( if self._cp_shard_fn is not None: und_seq, gen_seq, rotary_emb = self._cp_shard_fn(und_seq, gen_seq, rotary_emb) - und_out, gen_out = self._run_decoder_stack(und_seq, gen_seq, rotary_emb) + und_kv = None + if self._und_kv_cache is not None and self._cp_shard_fn is None and not torch.is_grad_enabled(): + # Keyed on the identity of the per-call text inputs. The entry holds references to them, so their ids + # cannot be reused by other tensors while the cache lives. One entry per CFG branch at most: if the + # inputs are new tensors on every step (e.g. copied by a hook), the extra calls run uncached. + key = (id(input_ids), id(position_ids), und_len) + entry = self._und_kv_cache.get(key) + if entry is None and len(self._und_kv_cache) < 2: + entry = self._und_kv_cache[key] = (input_ids, position_ids, [[] for _ in self.layers]) + if entry is not None: + und_kv = entry[2] + + if und_kv is None: + und_out, gen_out = self._run_decoder_stack(und_seq, gen_seq, rotary_emb) + else: + und_out, gen_out = self._run_decoder_stack(und_seq, gen_seq, rotary_emb, und_kv=und_kv) # Optional context-parallelism gather seam: re-gather the full per-pathway # sequence on every rank (and drop the padding) before the global-index decode @@ -824,6 +859,10 @@ def forward( if self._cp_gather_fn is not None: und_out, gen_out = self._cp_gather_fn(und_out, gen_out) + # The und pathway was reused from `und_kv`; predictions are only read at gen positions. + if und_out is None: + und_out = gen_out.new_zeros(und_len, gen_out.shape[-1]) + last_hidden_state = torch.cat([und_out, gen_out], dim=0) # Decode vision predictions from the joint hidden state. diff --git a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py index edd8f44409c2..9b374ce73373 100644 --- a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py +++ b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py @@ -30,6 +30,7 @@ from ...models.autoencoders.autoencoder_kl_wan import AutoencoderKLWan from ...models.modeling_utils import get_parameter_device from ...models.transformers.transformer_cosmos3 import ( + Cosmos3AttnProcessor, Cosmos3OmniTransformer, ) from ...schedulers import UniPCMultistepScheduler @@ -1715,6 +1716,14 @@ def __call__( ) self._mixed_precision_config = mixed_precision self._mixed_precision_trace = [] + # The text (und) pathway is the same on every step: let the transformer reuse its keys/values within this + # call. Not with step-dependent precision, cache hooks or custom attention processors. + if ( + not mixed_precision.enabled + and not self.transformer.is_cache_enabled + and all(type(p) is Cosmos3AttnProcessor for p in self.transformer.attn_processors.values()) + ): + self.transformer._und_kv_cache = {} try: with self.progress_bar(total=num_inference_steps) as progress_bar: for i, t in enumerate(timesteps): @@ -1892,6 +1901,7 @@ def __call__( progress_bar.update() finally: reset_cosmos3_mixed_precision(self.transformer, mixed_precision) + self.transformer._und_kv_cache = None self._current_timestep = None diff --git a/tests/models/transformers/test_models_transformer_cosmos3.py b/tests/models/transformers/test_models_transformer_cosmos3.py index c897d5bfdc69..755b3e6df92a 100644 --- a/tests/models/transformers/test_models_transformer_cosmos3.py +++ b/tests/models/transformers/test_models_transformer_cosmos3.py @@ -672,6 +672,87 @@ def test_cosmos3_nemotron_rms_norm_multiplies_in_float32(self): torch.testing.assert_close(norm(hidden_states), expected, rtol=0, atol=0) +class TestCosmos3OmniTransformerTextKVReuse(Cosmos3OmniTransformerTesterConfig): + """`_und_kv_cache` (set by `Cosmos3OmniPipeline` for one call) reuses the text-prefix keys/values across steps.""" + + num_steps = 3 + + def get_text_inputs(self, und_len: int, seed: int) -> dict: + # One text prefix per CFG branch; the same tensors are passed on every step, as in the pipeline. + generator = torch.Generator("cpu").manual_seed(seed) + sequence_length = und_len + 4 + return { + "input_ids": torch.randint(0, 32, (und_len,), generator=generator).to(torch_device), + "text_indexes": torch.arange(und_len, device=torch_device), + "position_ids": torch.randint(0, 16, (3, sequence_length), generator=generator).to(torch_device), + "und_len": und_len, + "sequence_length": sequence_length, + "vision_token_shapes": [(1, 2, 2)], + "vision_sequence_indexes": torch.arange(und_len, sequence_length, device=torch_device), + "vision_mse_loss_indexes": torch.arange(und_len, sequence_length, device=torch_device), + "vision_noisy_frame_indexes": [torch.tensor([0], device=torch_device)], + } + + def run_steps(self, model, branches, und_kv_cache): + model._und_kv_cache = und_kv_cache + outputs = [] + try: + for step in range(self.num_steps): + for branch_index, text_inputs in enumerate(branches): + generator = torch.Generator("cpu").manual_seed(10 * step + branch_index) + vision_tokens = randn_tensor((1, 2, 1, 2, 2), generator=generator, device=torch_device) + with torch.no_grad(): + output = model( + **text_inputs, + vision_tokens=[vision_tokens], + vision_timesteps=torch.full((4,), 1.0 - 0.3 * step, device=torch_device), + ) + outputs.append(output.sample[0]) + finally: + model._und_kv_cache = None + return outputs + + def get_model_and_branches(self, use_und_k_norm_for_gen, guidance): + torch.manual_seed(0) + model = self.model_class(**self.get_init_dict(), use_und_k_norm_for_gen=use_und_k_norm_for_gen) + model = model.to(torch_device).eval() + branches = [self.get_text_inputs(und_len=3, seed=1)] + if guidance: + branches.append(self.get_text_inputs(und_len=5, seed=2)) + return model, branches + + @pytest.mark.parametrize("guidance", [False, True], ids=["no_cfg", "cfg"]) + @pytest.mark.parametrize("use_und_k_norm_for_gen", [False, True]) + def test_text_kv_reuse_matches_full_pass(self, use_und_k_norm_for_gen, guidance): + model, branches = self.get_model_and_branches(use_und_k_norm_for_gen, guidance) + + recomputed = self.run_steps(model, branches, und_kv_cache=None) + reused = self.run_steps(model, branches, und_kv_cache={}) + + assert len(reused) == len(recomputed) == self.num_steps * len(branches) + for reused_output, recomputed_output in zip(reused, recomputed): + assert torch.equal(reused_output, recomputed_output) + + @pytest.mark.parametrize("guidance", [False, True], ids=["no_cfg", "cfg"]) + def test_text_kv_reuse_runs_text_pathway_once_per_branch(self, guidance): + model, branches = self.get_model_and_branches(use_und_k_norm_for_gen=True, guidance=guidance) + text_mlp_calls = 0 + + def count_text_mlp_calls(_module, _args, _output): + nonlocal text_mlp_calls + text_mlp_calls += 1 + + for layer in model.layers: + layer.mlp.register_forward_hook(count_text_mlp_calls) + + self.run_steps(model, branches, und_kv_cache=None) + assert text_mlp_calls == len(model.layers) * self.num_steps * len(branches) + + text_mlp_calls = 0 + self.run_steps(model, branches, und_kv_cache={}) + assert text_mlp_calls == len(model.layers) * len(branches) + + class TestCosmos3OmniTransformerMemory(Cosmos3OmniTransformerTesterConfig, MemoryTesterMixin): @pytest.mark.skip("The transformer returns one tensor list per generated modality.") def test_layerwise_casting_training(self): diff --git a/tests/pipelines/cosmos/test_cosmos3.py b/tests/pipelines/cosmos/test_cosmos3.py index 5660698405c3..55a154e68072 100644 --- a/tests/pipelines/cosmos/test_cosmos3.py +++ b/tests/pipelines/cosmos/test_cosmos3.py @@ -215,6 +215,31 @@ def test_i2v_pipeline_uses_native_preprocessing(self): torch.testing.assert_close(wide_output, crop_output) + @pytest.mark.parametrize("guidance_scale", [1.0, 2.0], ids=["no_cfg", "cfg"]) + def test_text_kv_reuse_matches_recompute(self, guidance_scale): + pipeline = self.get_pipeline().to(torch_device) + pipeline.set_progress_bar_config(disable=None) + + def run(prompt): + inputs = self.get_dummy_inputs() + inputs.update(prompt=prompt, guidance_scale=guidance_scale, num_inference_steps=3, output_type="latent") + return pipeline(**inputs).video + + # Two calls with different prompts: the text keys/values of the first call must not leak into the second. + prompts = ["a dog", "a cat sitting on a red mat"] + reused = [run(prompt) for prompt in prompts] + assert getattr(pipeline.transformer, "_und_kv_cache", None) is None + + # Reference: the text pathway recomputed on every step. + handle = pipeline.transformer.register_forward_pre_hook( + lambda module, _args: setattr(module, "_und_kv_cache", None) + ) + recomputed = [run(prompt) for prompt in prompts] + handle.remove() + + for reused_video, recomputed_video in zip(reused, recomputed): + assert torch.equal(reused_video, recomputed_video) + @pytest.mark.skip("Cosmos3 currently supports one prompt per pipeline call.") def test_inference_batch_consistent(self): pass From 9ea3e97b1ede2b62dec2b616f91a7253aa9d337b Mon Sep 17 00:00:00 2001 From: "Hsu, Chan Fu" Date: Wed, 18 Nov 2026 12:28:34 +0800 Subject: [PATCH 2/5] Cosmos3: pass the text-prefix KV cache explicitly (Cosmos3KVCache) Follow the QwenImage21KVCache / Flux2KVCache pattern instead of the private _und_kv_cache attribute keyed on tensor ids. Add Cosmos3KVLayerCache / Cosmos3KVCache. Cosmos3OmniTransformer.forward takes kv_cache and kv_cache_mode ("extract" stores each layer's post-RoPE und keys and values, "cached" reads them and skips the und pathway). Cosmos3OmniPipeline builds one cache per CFG branch behind a new use_kv_cache argument (default True), extracting on the first step; it stays off with mixed precision, cache / layer-skip hooks (also when added with apply_*), context parallelism and custom attention processors. Without a cache, _run_decoder_stack is called with three arguments as before, so cache hooks that replace it keep working. Tests drive kv_cache / kv_cache_mode directly, check that the pipeline uses the cache (text MLP call count) and matches use_kv_cache=False bit for bit, and that it falls back under apply_sea_cache. Signed-off-by: Hsu, Chan Fu --- .../transformers/transformer_cosmos3.py | 143 +++++++++++++----- .../pipelines/cosmos/pipeline_cosmos3_omni.py | 32 +++- .../test_models_transformer_cosmos3.py | 50 +++--- tests/pipelines/cosmos/test_cosmos3.py | 46 ++++-- 4 files changed, 191 insertions(+), 80 deletions(-) diff --git a/src/diffusers/models/transformers/transformer_cosmos3.py b/src/diffusers/models/transformers/transformer_cosmos3.py index d34debb83838..885ba117a89f 100644 --- a/src/diffusers/models/transformers/transformer_cosmos3.py +++ b/src/diffusers/models/transformers/transformer_cosmos3.py @@ -47,6 +47,37 @@ class Cosmos3OmniTransformerOutput(BaseOutput): action: list[torch.Tensor] | None = None +class Cosmos3KVLayerCache: + """Per-layer KV cache for the text (understanding) prefix. + + Stores the und keys (post-RoPE, as read by the generation pathway) and values extracted during the first denoising + step. Tensor format: ``(und_len, num_key_value_heads, head_dim)``. + """ + + def __init__(self): + self.k: torch.Tensor | None = None + self.v: torch.Tensor | None = None + + def store(self, k: torch.Tensor, v: torch.Tensor): + self.k = k + self.v = v + + def get(self) -> tuple[torch.Tensor, torch.Tensor]: + if self.k is None: + raise RuntimeError("KV cache has not been populated yet.") + return self.k, self.v + + +class Cosmos3KVCache: + """Container for all decoder layers' text-prefix KV caches.""" + + def __init__(self, num_layers: int): + self.layer_caches = [Cosmos3KVLayerCache() for _ in range(num_layers)] + + def get_layer(self, layer_idx: int) -> Cosmos3KVLayerCache: + return self.layer_caches[layer_idx] + + class Cosmos3AttnProcessor: """Dual-pathway attention processor for Cosmos3. @@ -63,16 +94,17 @@ def __call__( und_seq: torch.Tensor | None, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - und_kv: list[torch.Tensor] | None = None, + layer_cache: Cosmos3KVLayerCache | None = None, + kv_cache_mode: str | None = None, ) -> tuple[torch.Tensor | None, torch.Tensor]: cos_und, sin_und, cos_gen, sin_gen = rotary_emb # Causal pathway (understanding): und tokens self-attend with causal masking and never read gen tokens, so - # the und keys/values used by the gen pathway only depend on the text. When `und_kv` is given it is filled - # with them; when `und_seq` is None they are taken from `und_kv` and the und pathway is skipped. + # the und keys/values used by the gen pathway only depend on the text. "extract" stores them in + # `layer_cache`; "cached" reads them from it and skips the und pathway. und_out = None - if und_seq is None: - k_und_for_gen, v_und = und_kv + if kv_cache_mode == "cached": + k_und_for_gen, v_und = layer_cache.get() else: q_und = attn.to_q(und_seq).view(-1, attn.num_attention_heads, attn.head_dim) k_und = attn.to_k(und_seq).view(-1, attn.num_key_value_heads, attn.head_dim) @@ -95,8 +127,8 @@ def __call__( parallel_config=self._parallel_config, ) und_out = attn.to_out(causal_out.squeeze(0).flatten(-2, -1)) - if und_kv is not None: - und_kv[:] = [k_und_for_gen, v_und] + if kv_cache_mode == "extract": + layer_cache.store(k_und_for_gen, v_und) q_gen = attn.add_q_proj(gen_seq).view(-1, attn.num_attention_heads, attn.head_dim) k_gen = attn.add_k_proj(gen_seq).view(-1, attn.num_key_value_heads, attn.head_dim) @@ -300,11 +332,12 @@ def forward( und_seq: torch.Tensor | None, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - und_kv: list[torch.Tensor] | None = None, + layer_cache: Cosmos3KVLayerCache | None = None, + kv_cache_mode: str | None = None, ) -> tuple[torch.Tensor | None, torch.Tensor]: - if und_kv is None: + if kv_cache_mode is None: return self.processor(self, und_seq, gen_seq, rotary_emb) - return self.processor(self, und_seq, gen_seq, rotary_emb, und_kv=und_kv) + return self.processor(self, und_seq, gen_seq, rotary_emb, layer_cache=layer_cache, kv_cache_mode=kv_cache_mode) class Cosmos3VLTextMoTDecoderLayer(nn.Module): @@ -363,13 +396,16 @@ def forward( und_seq: torch.Tensor | None, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - und_kv: list[torch.Tensor] | None = None, + layer_cache: Cosmos3KVLayerCache | None = None, + kv_cache_mode: str | None = None, ) -> tuple[torch.Tensor | None, torch.Tensor]: - # `und_seq` is None when the und keys/values come from `und_kv` (see `Cosmos3AttnProcessor`). + # `und_seq` is None in "cached" mode: the und keys/values come from `layer_cache` (see `Cosmos3AttnProcessor`). und_norm = self.input_layernorm(und_seq) if und_seq is not None else None gen_norm = self.input_layernorm_moe_gen(gen_seq) - und_attn_out, gen_attn_out = self.self_attn(und_norm, gen_norm, rotary_emb, und_kv=und_kv) + und_attn_out, gen_attn_out = self.self_attn( + und_norm, gen_norm, rotary_emb, layer_cache=layer_cache, kv_cache_mode=kv_cache_mode + ) residual_gen = gen_seq + gen_attn_out mlp_out_gen = self.mlp_moe_gen(self.post_attention_layernorm_moe_gen(residual_gen)) if und_seq is None: @@ -399,11 +435,15 @@ class Cosmos3OmniTransformer(ModelMixin, ConfigMixin, PeftAdapterMixin, Attentio # expressed as diffusers' declarative `_cp_plan`, so CP lives outside the model. _cp_shard_fn = None _cp_gather_fn = None - # Optional reuse of the text (und) pathway across denoising steps. The und tokens never attend to the gen tokens, - # so for the same `input_ids` / `position_ids` every layer's und keys/values are the same on every step. - # `Cosmos3OmniPipeline` sets this to an empty dict for the duration of one `__call__`; `None` disables it. - _und_kv_cache = None - _skip_keys = ["und_kv"] + _skip_keys = ["kv_cache"] + # Hooks that only move or cast weights. Any other diffusers hook (cache, layer skip, guidance, context parallel) + # may skip, replace or reshard blocks, which `kv_cache` does not support. + _kv_cache_compatible_hooks = { + "group_offloading", + "lazy_prefetch_group_offloading", + "layerwise_casting", + "peft_autocast_disable", + } # `dtype` is injected into init_dict by ModelMixin.from_pretrained (configuration_utils.py:289), # so __init__ must accept it. Excluding it here keeps save_pretrained from writing it into # config.json — the value is a load-time runtime hint, not part of the model architecture. @@ -664,25 +704,35 @@ def _unpack_action_latents( unpacked.append(output) return unpacked + def _has_kv_cache_incompatible_hooks(self) -> bool: + # Also catches hooks added with `apply_*` functions, which do not set `_cache_config`. + for module in self.modules(): + registry = getattr(module, "_diffusers_hook", None) + if registry is not None and set(registry.hooks) - self._kv_cache_compatible_hooks: + return True + return False + def _run_decoder_stack( self, und_seq: torch.Tensor, gen_seq: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], - und_kv: list[list[torch.Tensor]] | None = None, + kv_cache: Cosmos3KVCache | None = None, + kv_cache_mode: str | None = None, ) -> tuple[torch.Tensor | None, torch.Tensor]: - # `und_kv`: one list per layer. Empty lists are filled by this pass; filled lists replace the und pathway. - if und_kv is not None and und_kv[-1]: + # In "cached" mode the und pathway is skipped: every layer reads the und keys/values from `kv_cache`. + if kv_cache_mode == "cached": und_seq = None - for i, decoder_layer in enumerate(self.layers): + for index_block, decoder_layer in enumerate(self.layers): + layer_cache = kv_cache.get_layer(index_block) if kv_cache is not None else None if torch.is_grad_enabled() and self.gradient_checkpointing: und_seq, gen_seq = self._gradient_checkpointing_func( - decoder_layer.__call__, und_seq, gen_seq, rotary_emb + decoder_layer.__call__, und_seq, gen_seq, rotary_emb, layer_cache, kv_cache_mode ) - elif und_kv is None: - und_seq, gen_seq = decoder_layer(und_seq, gen_seq, rotary_emb) else: - und_seq, gen_seq = decoder_layer(und_seq, gen_seq, rotary_emb, und_kv=und_kv[i]) + und_seq, gen_seq = decoder_layer( + und_seq, gen_seq, rotary_emb, layer_cache=layer_cache, kv_cache_mode=kv_cache_mode + ) return self.norm(und_seq) if und_seq is not None else None, self.norm_moe_gen(gen_seq) @@ -717,6 +767,8 @@ def forward( action_timesteps: torch.Tensor | None = None, action_noisy_frame_indexes: list[torch.Tensor] | None = None, action_domain_ids: list[torch.Tensor] | None = None, + kv_cache: Cosmos3KVCache | None = None, + kv_cache_mode: str | None = None, return_dict: bool = True, ) -> ( Cosmos3OmniTransformerOutput | tuple[list[torch.Tensor], list[torch.Tensor] | None, list[torch.Tensor] | None] @@ -748,12 +800,29 @@ def forward( action_timesteps: Optional per-token diffusion timesteps for action tokens. action_noisy_frame_indexes: Optional noisy frame indices per action item. action_domain_ids: Optional per-item domain IDs selecting the action head weights. + kv_cache: Optional ``Cosmos3KVCache`` holding the text-prefix keys/values. Pass together with + ``kv_cache_mode``. Valid because the und tokens never attend to the generation tokens, so their + keys/values only depend on ``input_ids`` and ``position_ids``. + kv_cache_mode: ``"extract"`` (run the und pathway and store its keys/values in ``kv_cache``) or + ``"cached"`` (skip the und pathway and read them from ``kv_cache``). Use one cache per text input, + e.g. one per CFG branch. return_dict: Whether to return a [`Cosmos3OmniTransformerOutput`] instead of a tuple. Returns: A [`Cosmos3OmniTransformerOutput`] or a tuple of per-modality prediction lists. Optional modalities return ``None`` when their inputs are omitted. """ + if kv_cache is not None and kv_cache_mode not in ("extract", "cached"): + raise ValueError( + f"kv_cache_mode must be 'extract' or 'cached' when kv_cache is provided, got {kv_cache_mode!r}." + ) + if kv_cache is None and kv_cache_mode is not None: + raise ValueError(f"kv_cache_mode is {kv_cache_mode!r} but no kv_cache was passed to hold the prefix.") + if kv_cache is not None and self._cp_shard_fn is not None: + raise ValueError("kv_cache is not supported with context parallelism.") + if kv_cache is not None and self._has_kv_cache_incompatible_hooks(): + raise ValueError("kv_cache cannot be combined with cache, layer-skip or context-parallel hooks.") + has_sound = sound_tokens is not None and sound_sequence_indexes is not None has_action = action_tokens is not None and action_sequence_indexes is not None @@ -838,22 +907,14 @@ def forward( if self._cp_shard_fn is not None: und_seq, gen_seq, rotary_emb = self._cp_shard_fn(und_seq, gen_seq, rotary_emb) - und_kv = None - if self._und_kv_cache is not None and self._cp_shard_fn is None and not torch.is_grad_enabled(): - # Keyed on the identity of the per-call text inputs. The entry holds references to them, so their ids - # cannot be reused by other tensors while the cache lives. One entry per CFG branch at most: if the - # inputs are new tensors on every step (e.g. copied by a hook), the extra calls run uncached. - key = (id(input_ids), id(position_ids), und_len) - entry = self._und_kv_cache.get(key) - if entry is None and len(self._und_kv_cache) < 2: - entry = self._und_kv_cache[key] = (input_ids, position_ids, [[] for _ in self.layers]) - if entry is not None: - und_kv = entry[2] - - if und_kv is None: + # Cache hooks (e.g. SeaCache) replace `_run_decoder_stack` with a 3-argument version, so the KV-cache + # arguments are only passed when a cache is used (rejected above together with such hooks). + if kv_cache is None: und_out, gen_out = self._run_decoder_stack(und_seq, gen_seq, rotary_emb) else: - und_out, gen_out = self._run_decoder_stack(und_seq, gen_seq, rotary_emb, und_kv=und_kv) + und_out, gen_out = self._run_decoder_stack( + und_seq, gen_seq, rotary_emb, kv_cache=kv_cache, kv_cache_mode=kv_cache_mode + ) # Optional context-parallelism gather seam: re-gather the full per-pathway # sequence on every rank (and drop the padding) before the global-index decode @@ -862,7 +923,7 @@ def forward( if self._cp_gather_fn is not None: und_out, gen_out = self._cp_gather_fn(und_out, gen_out) - # The und pathway was reused from `und_kv`; predictions are only read at gen positions. + # The und pathway was skipped ("cached" mode); predictions are only read at gen positions. if und_out is None: und_out = gen_out.new_zeros(und_len, gen_out.shape[-1]) diff --git a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py index b1939a3d30c3..0f33582b52e8 100644 --- a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py +++ b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py @@ -31,6 +31,7 @@ from ...models.modeling_utils import get_parameter_device from ...models.transformers.transformer_cosmos3 import ( Cosmos3AttnProcessor, + Cosmos3KVCache, Cosmos3OmniTransformer, ) from ...schedulers import UniPCMultistepScheduler @@ -1351,6 +1352,7 @@ def __call__( mixed_precision_first_steps: int | None = None, mixed_precision_last_steps: int | None = None, mixed_precision_reasoner_policy: str | None = None, + use_kv_cache: bool = True, ) -> Cosmos3OmniPipelineOutput | tuple: r""" Run the Cosmos 3 omni pipeline end-to-end: encode the (optional) conditioning image/video, denoise vision and @@ -1456,6 +1458,11 @@ def __call__( Optional trailing W8A16 step count. mixed_precision_reasoner_policy (`str`, *optional*): Optional reasoner path: `"high_precision"` (W8A16) or `"base_precision"` (native W8A8). + use_kv_cache (`bool`, *optional*, defaults to `True`): + Cache the text-prefix keys and values of each CFG branch after the first step and skip the text + pathway on later steps. Valid because the text tokens never attend to the video, sound or action + tokens, so their keys and values are the same on every step. Not used with mixed precision, cache + or layer-skip hooks, context parallelism or custom attention processors. Returns: [`Cosmos3OmniPipelineOutput`] or `tuple`: @@ -1716,14 +1723,19 @@ def __call__( ) self._mixed_precision_config = mixed_precision self._mixed_precision_trace = [] - # The text (und) pathway is the same on every step: let the transformer reuse its keys/values within this - # call. Not with step-dependent precision, cache hooks or custom attention processors. - if ( - not mixed_precision.enabled - and not self.transformer.is_cache_enabled + # The text (und) keys and values are the same on every step, so the first step extracts them and later steps + # only run the generation pathway. Not with step-dependent precision, cache / layer-skip hooks, context + # parallelism or custom attention processors. + cache_enabled = ( + use_kv_cache + and not mixed_precision.enabled + and not self.transformer._has_kv_cache_incompatible_hooks() + and self.transformer._cp_shard_fn is None and all(type(p) is Cosmos3AttnProcessor for p in self.transformer.attn_processors.values()) - ): - self.transformer._und_kv_cache = {} + ) + num_layers = len(self.transformer.layers) + cond_kv_cache = Cosmos3KVCache(num_layers) if cache_enabled else None + uncond_kv_cache = Cosmos3KVCache(num_layers) if cache_enabled and self.do_classifier_free_guidance else None try: with self.progress_bar(total=num_inference_steps) as progress_bar: for i, t in enumerate(timesteps): @@ -1738,6 +1750,7 @@ def __call__( continue self._current_timestep = t + kv_cache_mode = None if cond_kv_cache is None else ("extract" if i == 0 else "cached") sigma = float(self.scheduler.sigmas[i]) timestep = t.item() @@ -1787,6 +1800,8 @@ def __call__( action_timesteps=action_timesteps, action_noisy_frame_indexes=cond_packed_static.get("action_noisy_frame_indexes"), action_domain_ids=[action_domain_id] if action_domain_id is not None else None, + kv_cache=cond_kv_cache, + kv_cache_mode=kv_cache_mode, return_dict=False, ) cond_v_vision, cond_v_sound, cond_v_action = self._mask_velocity_predictions( @@ -1830,6 +1845,8 @@ def __call__( action_timesteps=action_timesteps, action_noisy_frame_indexes=uncond_packed_static.get("action_noisy_frame_indexes"), action_domain_ids=[action_domain_id] if action_domain_id is not None else None, + kv_cache=uncond_kv_cache, + kv_cache_mode=kv_cache_mode, return_dict=False, ) uncond_v_vision, uncond_v_sound, uncond_v_action = self._mask_velocity_predictions( @@ -1901,7 +1918,6 @@ def __call__( progress_bar.update() finally: reset_cosmos3_mixed_precision(self.transformer, mixed_precision) - self.transformer._und_kv_cache = None self._current_timestep = None diff --git a/tests/models/transformers/test_models_transformer_cosmos3.py b/tests/models/transformers/test_models_transformer_cosmos3.py index 1bdfc0de14d1..78a055e27c9a 100644 --- a/tests/models/transformers/test_models_transformer_cosmos3.py +++ b/tests/models/transformers/test_models_transformer_cosmos3.py @@ -22,6 +22,7 @@ from diffusers.hooks.sea_cache import _SEA_CACHE_ROOT_HOOK from diffusers.models.cache_utils import CacheMixin from diffusers.models.transformers.transformer_cosmos3 import ( + Cosmos3KVCache, Cosmos3NemotronRMSNorm, Cosmos3OmniTransformerOutput, Cosmos3PackedMoTAttention, @@ -677,7 +678,7 @@ def test_cosmos3_nemotron_rms_norm_multiplies_in_float32(self): class TestCosmos3OmniTransformerTextKVReuse(Cosmos3OmniTransformerTesterConfig): - """`_und_kv_cache` (set by `Cosmos3OmniPipeline` for one call) reuses the text-prefix keys/values across steps.""" + """`kv_cache` / `kv_cache_mode` reuse the text-prefix keys/values across denoising steps.""" num_steps = 3 @@ -697,23 +698,24 @@ def get_text_inputs(self, und_len: int, seed: int) -> dict: "vision_noisy_frame_indexes": [torch.tensor([0], device=torch_device)], } - def run_steps(self, model, branches, und_kv_cache): - model._und_kv_cache = und_kv_cache + def run_steps(self, model, branches, use_kv_cache): + # One cache per CFG branch, as in the pipeline: "extract" on the first step, "cached" afterwards. + kv_caches = [Cosmos3KVCache(len(model.layers)) if use_kv_cache else None for _ in branches] outputs = [] - try: - for step in range(self.num_steps): - for branch_index, text_inputs in enumerate(branches): - generator = torch.Generator("cpu").manual_seed(10 * step + branch_index) - vision_tokens = randn_tensor((1, 2, 1, 2, 2), generator=generator, device=torch_device) - with torch.no_grad(): - output = model( - **text_inputs, - vision_tokens=[vision_tokens], - vision_timesteps=torch.full((4,), 1.0 - 0.3 * step, device=torch_device), - ) - outputs.append(output.sample[0]) - finally: - model._und_kv_cache = None + for step in range(self.num_steps): + kv_cache_mode = ("extract" if step == 0 else "cached") if use_kv_cache else None + for branch_index, (text_inputs, kv_cache) in enumerate(zip(branches, kv_caches)): + generator = torch.Generator("cpu").manual_seed(10 * step + branch_index) + vision_tokens = randn_tensor((1, 2, 1, 2, 2), generator=generator, device=torch_device) + with torch.no_grad(): + output = model( + **text_inputs, + vision_tokens=[vision_tokens], + vision_timesteps=torch.full((4,), 1.0 - 0.3 * step, device=torch_device), + kv_cache=kv_cache, + kv_cache_mode=kv_cache_mode, + ) + outputs.append(output.sample[0]) return outputs def get_model_and_branches(self, use_und_k_norm_for_gen, guidance): @@ -730,8 +732,8 @@ def get_model_and_branches(self, use_und_k_norm_for_gen, guidance): def test_text_kv_reuse_matches_full_pass(self, use_und_k_norm_for_gen, guidance): model, branches = self.get_model_and_branches(use_und_k_norm_for_gen, guidance) - recomputed = self.run_steps(model, branches, und_kv_cache=None) - reused = self.run_steps(model, branches, und_kv_cache={}) + recomputed = self.run_steps(model, branches, use_kv_cache=False) + reused = self.run_steps(model, branches, use_kv_cache=True) assert len(reused) == len(recomputed) == self.num_steps * len(branches) for reused_output, recomputed_output in zip(reused, recomputed): @@ -749,13 +751,19 @@ def count_text_mlp_calls(_module, _args, _output): for layer in model.layers: layer.mlp.register_forward_hook(count_text_mlp_calls) - self.run_steps(model, branches, und_kv_cache=None) + self.run_steps(model, branches, use_kv_cache=False) assert text_mlp_calls == len(model.layers) * self.num_steps * len(branches) text_mlp_calls = 0 - self.run_steps(model, branches, und_kv_cache={}) + self.run_steps(model, branches, use_kv_cache=True) assert text_mlp_calls == len(model.layers) * len(branches) + def test_kv_cache_mode_requires_kv_cache(self): + model, branches = self.get_model_and_branches(use_und_k_norm_for_gen=True, guidance=False) + vision_tokens = randn_tensor((1, 2, 1, 2, 2), device=torch_device) + with pytest.raises(ValueError, match="no kv_cache"): + model(**branches[0], vision_tokens=[vision_tokens], vision_timesteps=torch.ones(4), kv_cache_mode="cached") + class TestCosmos3OmniTransformerMemory(Cosmos3OmniTransformerTesterConfig, MemoryTesterMixin): @pytest.mark.skip("The transformer returns one tensor list per generated modality.") diff --git a/tests/pipelines/cosmos/test_cosmos3.py b/tests/pipelines/cosmos/test_cosmos3.py index 55a154e68072..39cc3759cd0e 100644 --- a/tests/pipelines/cosmos/test_cosmos3.py +++ b/tests/pipelines/cosmos/test_cosmos3.py @@ -25,7 +25,9 @@ AutoencoderKLWan, Cosmos3OmniPipeline, Cosmos3OmniTransformer, + SeaCacheConfig, UniPCMultistepScheduler, + apply_sea_cache, ) from diffusers.pipelines.cosmos.pipeline_cosmos3_omni import _preprocess_conditioning_image @@ -220,26 +222,50 @@ def test_text_kv_reuse_matches_recompute(self, guidance_scale): pipeline = self.get_pipeline().to(torch_device) pipeline.set_progress_bar_config(disable=None) - def run(prompt): + def run(prompt, use_kv_cache): inputs = self.get_dummy_inputs() - inputs.update(prompt=prompt, guidance_scale=guidance_scale, num_inference_steps=3, output_type="latent") + inputs.update( + prompt=prompt, + guidance_scale=guidance_scale, + num_inference_steps=3, + output_type="latent", + use_kv_cache=use_kv_cache, + ) return pipeline(**inputs).video + # Count the text-pathway MLP calls of one layer to check that the cache is actually used. + text_mlp_calls = 0 + + def count_text_mlp_calls(_module, _args, _output): + nonlocal text_mlp_calls + text_mlp_calls += 1 + + pipeline.transformer.layers[0].mlp.register_forward_hook(count_text_mlp_calls) + num_branches = 2 if guidance_scale > 1.0 else 1 + # Two calls with different prompts: the text keys/values of the first call must not leak into the second. prompts = ["a dog", "a cat sitting on a red mat"] - reused = [run(prompt) for prompt in prompts] - assert getattr(pipeline.transformer, "_und_kv_cache", None) is None - + reused = [run(prompt, use_kv_cache=True) for prompt in prompts] + assert text_mlp_calls == len(prompts) * num_branches # Reference: the text pathway recomputed on every step. - handle = pipeline.transformer.register_forward_pre_hook( - lambda module, _args: setattr(module, "_und_kv_cache", None) - ) - recomputed = [run(prompt) for prompt in prompts] - handle.remove() + text_mlp_calls = 0 + recomputed = [run(prompt, use_kv_cache=False) for prompt in prompts] + assert text_mlp_calls == len(prompts) * num_branches * 3 for reused_video, recomputed_video in zip(reused, recomputed): assert torch.equal(reused_video, recomputed_video) + def test_text_kv_reuse_off_with_cache_hooks(self): + # Hooks added with `apply_*` (not only `enable_cache`) must turn the text KV cache off instead of failing. + pipeline = self.get_pipeline().to(torch_device) + pipeline.set_progress_bar_config(disable=None) + apply_sea_cache(pipeline.transformer, SeaCacheConfig()) + assert pipeline.transformer._has_kv_cache_incompatible_hooks() + + inputs = self.get_dummy_inputs() + inputs.update(guidance_scale=2.0, num_inference_steps=3, output_type="latent") + assert pipeline(**inputs).video is not None + @pytest.mark.skip("Cosmos3 currently supports one prompt per pipeline call.") def test_inference_batch_consistent(self): pass From c1fe31c5178c037dad19c516b15b4a1c98acbd2a Mon Sep 17 00:00:00 2001 From: "Hsu, Chan Fu" Date: Wed, 18 Nov 2026 14:07:13 +0800 Subject: [PATCH 3/5] Cosmos3: free the text KV caches before decoding Drop the cond / uncond caches when the denoising loop ends so they do not raise the peak memory of the VAE decode. Signed-off-by: Hsu, Chan Fu --- src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py index 0f33582b52e8..8c6ffba78d1c 100644 --- a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py +++ b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py @@ -1918,6 +1918,8 @@ def __call__( progress_bar.update() finally: reset_cosmos3_mixed_precision(self.transformer, mixed_precision) + # Free the cached keys/values before decoding so they do not raise the peak memory. + cond_kv_cache = uncond_kv_cache = None self._current_timestep = None From f124425c4e827cb9dea25c99b267e0956e4a4210 Mon Sep 17 00:00:00 2001 From: "Hsu, Chan Fu" Date: Wed, 18 Nov 2026 15:30:38 +0800 Subject: [PATCH 4/5] Cosmos3: no kv_cache_mode for the uncond pass without its cache If a step-end callback turns CFG on mid-loop, the uncond pass has no cache; run it uncached instead of raising. Signed-off-by: Hsu, Chan Fu --- src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py index 8c6ffba78d1c..35f066b79c1d 100644 --- a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py +++ b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py @@ -1846,7 +1846,7 @@ def __call__( action_noisy_frame_indexes=uncond_packed_static.get("action_noisy_frame_indexes"), action_domain_ids=[action_domain_id] if action_domain_id is not None else None, kv_cache=uncond_kv_cache, - kv_cache_mode=kv_cache_mode, + kv_cache_mode=kv_cache_mode if uncond_kv_cache is not None else None, return_dict=False, ) uncond_v_vision, uncond_v_sound, uncond_v_action = self._mask_velocity_predictions( From a1ed7a5638de19be43d922360db8870c3dd595c4 Mon Sep 17 00:00:00 2001 From: "Hsu, Chan Fu" Date: Wed, 18 Nov 2026 16:51:26 +0800 Subject: [PATCH 5/5] Cosmos3: clear the text KV caches after the denoising loop Follow Flux2 Klein KV / GlmImage: add clear() to Cosmos3KVLayerCache / Cosmos3KVCache and call it after the loop, before decoding, instead of dropping the references in the finally block. Test that both caches are cleared. Signed-off-by: Hsu, Chan Fu --- .../models/transformers/transformer_cosmos3.py | 8 ++++++++ .../pipelines/cosmos/pipeline_cosmos3_omni.py | 8 ++++++-- tests/pipelines/cosmos/test_cosmos3.py | 13 ++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/diffusers/models/transformers/transformer_cosmos3.py b/src/diffusers/models/transformers/transformer_cosmos3.py index 885ba117a89f..89a169d96712 100644 --- a/src/diffusers/models/transformers/transformer_cosmos3.py +++ b/src/diffusers/models/transformers/transformer_cosmos3.py @@ -67,6 +67,10 @@ def get(self) -> tuple[torch.Tensor, torch.Tensor]: raise RuntimeError("KV cache has not been populated yet.") return self.k, self.v + def clear(self): + self.k = None + self.v = None + class Cosmos3KVCache: """Container for all decoder layers' text-prefix KV caches.""" @@ -77,6 +81,10 @@ def __init__(self, num_layers: int): def get_layer(self, layer_idx: int) -> Cosmos3KVLayerCache: return self.layer_caches[layer_idx] + def clear(self): + for cache in self.layer_caches: + cache.clear() + class Cosmos3AttnProcessor: """Dual-pathway attention processor for Cosmos3. diff --git a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py index 35f066b79c1d..b85ea4287a04 100644 --- a/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py +++ b/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py @@ -1918,8 +1918,12 @@ def __call__( progress_bar.update() finally: reset_cosmos3_mixed_precision(self.transformer, mixed_precision) - # Free the cached keys/values before decoding so they do not raise the peak memory. - cond_kv_cache = uncond_kv_cache = None + + # Clean up KV cache + if cond_kv_cache is not None: + cond_kv_cache.clear() + if uncond_kv_cache is not None: + uncond_kv_cache.clear() self._current_timestep = None diff --git a/tests/pipelines/cosmos/test_cosmos3.py b/tests/pipelines/cosmos/test_cosmos3.py index 39cc3759cd0e..9ea9bdf00626 100644 --- a/tests/pipelines/cosmos/test_cosmos3.py +++ b/tests/pipelines/cosmos/test_cosmos3.py @@ -29,6 +29,7 @@ UniPCMultistepScheduler, apply_sea_cache, ) +from diffusers.models.transformers.transformer_cosmos3 import Cosmos3KVCache from diffusers.pipelines.cosmos.pipeline_cosmos3_omni import _preprocess_conditioning_image from ...testing_utils import enable_full_determinism, torch_device @@ -245,7 +246,17 @@ def count_text_mlp_calls(_module, _args, _output): # Two calls with different prompts: the text keys/values of the first call must not leak into the second. prompts = ["a dog", "a cat sitting on a red mat"] - reused = [run(prompt, use_kv_cache=True) for prompt in prompts] + # The caches are cleared after the denoising loop, before decoding. + original_clear = Cosmos3KVCache.clear + cleared = [] + + def tracking_clear(cache): + original_clear(cache) + cleared.append(all(layer.k is None and layer.v is None for layer in cache.layer_caches)) + + with mock.patch.object(Cosmos3KVCache, "clear", tracking_clear): + reused = [run(prompt, use_kv_cache=True) for prompt in prompts] + assert cleared == [True] * (len(prompts) * num_branches) assert text_mlp_calls == len(prompts) * num_branches # Reference: the text pathway recomputed on every step. text_mlp_calls = 0