@@ -96,7 +96,8 @@ def mask_mod(batch_idx, head_idx, q_idx, kv_idx):
9696# A keyframe decode carries a second stream through the decoder: a stack of single-frame latent *planes* `(B, P, H,
9797# W, C)` whose plane axis sits in the video's temporal slot. Every weight is shared between the two streams, and they
9898# only mix inside one joint neighborhood-attention softmax, where each video query also sees the same spatial window
99- # on its two nearest planes and each plane query also sees its two nearest video frames.
99+ # on its two nearest planes (only the first plane, for frames before it) and each plane query also sees its two nearest
100+ # video frames.
100101# --------------------------------------------------------------------------------------------------------------------
101102
102103# Keyframe planes visible to one video query, and video frames visible to one plane query.
@@ -554,9 +555,9 @@ def _joint_neighborhood_attention(
554555 """Joint neighborhood attention over a video volume and a stack of keyframe planes, one softmax per query.
555556
556557 A video query at `(t, h, w)` attends to its `Kt x Kh x Kw` video window plus the `Kh x Kw` window at the same `(h,
557- w)` on each of its `_KEYFRAME_CONTEXT_SLOTS` nearest planes, ranked by `|t - keyframe_times|` whatever `Kt` is. A
558- plane query attends to the `Kh x Kw` window on its own plane (there is no plane-to-plane attention) plus the same
559- window on each of its nearest video frames.
558+ w)` on each of its `_KEYFRAME_CONTEXT_SLOTS` nearest planes, ranked by `|t - keyframe_times|` whatever `Kt` is; a
559+ frame before the first plane sees that plane only. A plane query attends to the `Kh x Kw` window on its own plane
560+ (there is no plane-to-plane attention) plus the same window on each of its nearest video frames.
560561
561562 Unlike [`LTX2VideoVaeNeighborhoodAttnProcessor`] and NATTEN, whose windows shift inward at the volume border, every
562563 window here is centered and *clamped*: offsets that fall outside the volume are masked out. That is also why no
@@ -575,6 +576,8 @@ def _joint_neighborhood_attention(
575576 keyframe_times = keyframe_times .to (device = query .device , dtype = torch .float32 )
576577 frame_times = torch .arange (num_frames , dtype = torch .float32 , device = query .device )
577578 video_slots = _nearest_slots (frame_times , keyframe_times , _KEYFRAME_CONTEXT_SLOTS )
579+ # The reference decoder gives frames before the first plane that plane alone, not their two nearest.
580+ video_slots [frame_times < keyframe_times .min (), 1 :] = - 1
578581 keyframe_slots = _nearest_slots (keyframe_times , frame_times , _KEYFRAME_CONTEXT_SLOTS )
579582
580583 side = max (1 , round (math .sqrt (_JOINT_BRICK_QUERIES )))
@@ -1488,9 +1491,10 @@ class LTX2VideoDiffusionDecoderModel(ModelMixin, AttentionMixin, ConfigMixin):
14881491
14891492 Decoding can be anchored on *keyframe planes*: single-frame latents at known pixel frames, each encoded as a
14901493 standalone one-frame clip, passed as `keyframe_latents` / `keyframe_frame_indices` to [`decode`]. Every video
1491- position then also attends to the same spatial window on its two nearest planes. Checkpoints trained for it carry a
1492- learned tag added to the plane latents, `decoder.type_emb`, which `decoder_keyframe_type_embedding=True` creates.
1493- Decoding with keyframe planes on a checkpoint that lacks that tag raises rather than decoding them untagged.
1494+ position then also attends to the same spatial window on its two nearest planes, or on the first plane alone for
1495+ frames before it. Checkpoints trained for it carry a learned tag added to the plane latents, `decoder.type_emb`,
1496+ which `decoder_keyframe_type_embedding=True` creates. Decoding with keyframe planes on a checkpoint that lacks that
1497+ tag raises rather than decoding them untagged.
14941498
14951499 This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented
14961500 for all models (such as downloading or saving).
@@ -1921,7 +1925,7 @@ def forward(
19211925 keyframe_latents (`torch.Tensor`, *optional*):
19221926 Keyframe planes of shape `(B, C, P, H, W)`, denormalized like `z` and on its latent grid, one latent
19231927 frame per plane, each encoded as a standalone one-frame clip. Every video position also attends to the
1924- same spatial window on its two nearest planes.
1928+ same spatial window on its two nearest planes, or on the first plane alone for frames before it .
19251929 keyframe_frame_indices (`torch.Tensor`, *optional*):
19261930 The `(P,)` pixel frame of each plane in the decoded video. Required with `keyframe_latents`.
19271931
0 commit comments