diff --git a/assets/animatediff/img2video_demo.gif b/assets/animatediff/img2video_demo.gif new file mode 100644 index 000000000..2479596e3 Binary files /dev/null and b/assets/animatediff/img2video_demo.gif differ diff --git a/docs/animatediff.md b/docs/animatediff.md index b26b6aaee..cf71a0744 100644 --- a/docs/animatediff.md +++ b/docs/animatediff.md @@ -134,6 +134,25 @@ sd-cli -M vid_gen --model realisticVisionV60B1.safetensors \ -p "close up photo of a rabbit ..." ... ``` +## img2video + +Pass a pre-rendered image via `-i / --init-img` to animate FROM it. All N output frames start from the encoded init latent, then per-frame noise is added at `--strength`. Character identity, composition, and quality are anchored by the init image; the motion module adds subtle motion on top. + +Left: init image rendered with `-M img_gen`. Right: 8-frame vid_gen output. + + + +``` +sd-cli -M img_gen ... -o init.png # any high-quality still +sd-cli -M vid_gen --motion-module mm_sd15_v3.safetensors \ + -i init.png --strength 0.75 \ + --cfg-scale 7.0 --sampling-method euler --scheduler karras \ + -H 512 -W 512 --video-frames 8 --steps 25 -s 42 \ + -p "..." -o out.avi +``` + +`--strength` controls how far the motion module is allowed to deviate from the init image (higher = more motion, lower = more static). + ## Notes - The motion module was trained at `video_length=16`. Running with diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 10fdde11c..7e1a8e86d 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -4687,8 +4687,16 @@ static std::optional prepare_image_generation_latents(sd init_latent.dim() >= 4 && init_latent.shape()[3] == 1) { int n_frames = sd_ctx->sd->animatediff_num_frames; std::vector shape(init_latent.shape().begin(), init_latent.shape().end()); - shape[3] = n_frames; - init_latent = sd::Tensor(std::move(shape)); // zero-filled batch of N frames; per-frame noise is generated later via randn_like. + shape[3] = n_frames; + if (!init_image_tensor.empty()) { + sd::Tensor replicated(shape); + for (int f = 0; f < n_frames; ++f) { + sd::ops::slice_assign(&replicated, 3, f, f + 1, init_latent); + } + init_latent = std::move(replicated); + } else { + init_latent = sd::Tensor(std::move(shape)); + } } if (!control_image_tensor.empty()) { @@ -6132,6 +6140,7 @@ static bool generate_animatediff_video(sd_ctx_t* sd_ctx, img_gen_params.height = sd_vid_gen_params->height; img_gen_params.sample_params = sd_vid_gen_params->sample_params; img_gen_params.strength = sd_vid_gen_params->strength; + img_gen_params.init_image = sd_vid_gen_params->init_image; img_gen_params.seed = sd_vid_gen_params->seed; img_gen_params.batch_count = 1; img_gen_params.control_strength = 1.0f;