End stillness for start & end

app.py

@@ -373,7 +373,7 @@ def image_encode(image_np, target_width, target_height, vae, image_encoder, feat
         raise
 
 @torch.no_grad()
-def worker(input_image, end_image, image_position, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
+def worker(input_image, end_image, image_position, end_stillness, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
     def encode_prompt(prompt, n_prompt):
         llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
 
@@ -643,7 +643,7 @@ def worker(input_image, end_image, image_position, prompts, n_prompt, seed, reso
     return
 
 @torch.no_grad()
-def worker_start_end(input_image, end_image, image_position, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
+def worker_start_end(input_image, end_image, image_position, end_stillness, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
     def encode_prompt(prompt, n_prompt):
         llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
 
@@ -766,6 +766,7 @@ def worker_start_end(input_image, end_image, image_position, prompts, n_prompt,
         start_latent = start_latent.to(history_latents)
         if has_end_image:
             end_latent = end_latent.to(history_latents)
+            end_latent = end_latent.expand(-1, -1, 1 + end_stillness, -1, -1)
             
         history_pixels = None
         total_generated_latent_frames = 0
@@ -850,8 +851,8 @@ def worker_start_end(input_image, end_image, image_position, prompts, n_prompt,
             if len(prompt_parameters) > 0:
                 [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(len(prompt_parameters) - 1)
 
-            indices = torch.arange(1 + latent_padding_size + latent_window_size + 1 + 2 + 16).unsqueeze(0)
-            clean_latent_indices_pre, blank_indices, latent_indices, clean_latent_indices_post, clean_latent_2x_indices, clean_latent_4x_indices = indices.split([1, latent_padding_size, latent_window_size, 1, 2, 16], dim=1)
+            indices = torch.arange(1 + latent_padding_size + latent_window_size + 1 + (end_stillness if is_first_section else 0) + 2 + 16).unsqueeze(0)
+            clean_latent_indices_pre, blank_indices, latent_indices, clean_latent_indices_post, clean_latent_2x_indices, clean_latent_4x_indices = indices.split([1, latent_padding_size, latent_window_size, 1 + (end_stillness if is_first_section else 0), 2, 16], dim=1)
             clean_latent_indices = torch.cat([clean_latent_indices_pre, clean_latent_indices_post], dim=1)
             
             clean_latents_post, clean_latents_2x, clean_latents_4x = history_latents[:, :, :1 + 2 + 16, :, :].split([1, 2, 16], dim=2)
@@ -1225,17 +1226,17 @@ def worker_video(input_video, end_frame, end_stillness, prompts, n_prompt, seed,
     stream.output_queue.push(('end', None))
     return
 
-def get_duration(input_image, end_image, image_position, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
+def get_duration(input_image, end_image, image_position, end_stillness, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
     return allocation_time
 
 @spaces.GPU(duration=get_duration)
-def process_on_gpu(input_image, end_image, image_position, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number
+def process_on_gpu(input_image, end_image, image_position, end_stillness, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number
            ):
     start = time.time()
     global stream
     stream = AsyncStream()
 
-    async_run(worker_start_end if generation_mode == "start_end" else worker, input_image, end_image, image_position, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number)
+    async_run(worker_start_end if generation_mode == "start_end" else worker, input_image, end_image, image_position, end_stillness, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number)
 
     output_filename = None
 
@@ -1267,6 +1268,7 @@ def process_on_gpu(input_image, end_image, image_position, prompts, generation_m
 def process(input_image,
             end_image,
             image_position=0,
+            end_stillness=1,
             prompt="",
             generation_mode="image",
             n_prompt="",
@@ -1320,6 +1322,7 @@ def process(input_image,
     yield from process_on_gpu(input_image,
             end_image,
             image_position,
+            end_stillness,
             prompts,
             generation_mode,
             n_prompt,
@@ -1582,7 +1585,7 @@ with block:
             progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
             progress_bar = gr.HTML('', elem_classes='no-generating-animation')
 
-    ips = [input_image, end_image, image_position, final_prompt, generation_mode, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number]
+    ips = [input_image, end_image, image_position, end_stillness, final_prompt, generation_mode, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number]
     ips_video = [input_video, end_image, end_stillness, final_prompt, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch]
 
     with gr.Row(elem_id="text_examples", visible=False):
@@ -1593,6 +1596,7 @@ with block:
                         None, # input_image
                         None, # end_image
                         0, # image_position
+                        1, # end_stillness
                         "Overcrowed street in Japan, photorealistic, realistic, intricate details, 8k, insanely detailed",
                         "text", # generation_mode
                         "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
@@ -1629,6 +1633,7 @@ with block:
                         "./img_examples/Example2.webp", # input_image
                         None, # end_image
                         0, # image_position
+                        1, # end_stillness
                         "A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks, the man stops talking and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens",
                         "image", # generation_mode
                         "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
@@ -1653,6 +1658,7 @@ with block:
                         "./img_examples/Example1.png", # input_image
                         None, # end_image
                         0, # image_position
+                        1, # end_stillness
                         "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
                         "image", # generation_mode
                         "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
@@ -1677,6 +1683,7 @@ with block:
                         "./img_examples/Example4.webp", # input_image
                         None, # end_image
                         1, # image_position
+                        1, # end_stillness
                         "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
                         "image", # generation_mode
                         "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
@@ -1701,6 +1708,7 @@ with block:
                         "./img_examples/Example4.webp", # input_image
                         None, # end_image
                         50, # image_position
+                        1, # end_stillness
                         "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
                         "image", # generation_mode
                         "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
@@ -1725,6 +1733,7 @@ with block:
                         "./img_examples/Example4.webp", # input_image
                         None, # end_image
                         100, # image_position
+                        1, # end_stillness
                         "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
                         "image", # generation_mode
                         "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
@@ -1761,6 +1770,7 @@ with block:
                         "./img_examples/Example5.png", # input_image
                         "./img_examples/Example6.png", # end_image
                         0, # image_position
+                        1, # end_stillness
                         "A woman jumps out of the train and arrives on the ground, viewed from the outside, photorealistic, realistic, amateur photography, midday, insanely detailed, 8k",
                         "start_end", # generation_mode
                         "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth, jumpcut, crossfader, crossfading", # n_prompt
@@ -1886,6 +1896,7 @@ with block:
                     None, # input_image
                     None, # end_image
                     0, # image_position
+                    1, # end_stillness
                     "Overcrowed street in Japan, photorealistic, realistic, intricate details, 8k, insanely detailed",
                     "text", # generation_mode
                     "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
@@ -1921,6 +1932,7 @@ with block:
                     "./img_examples/Example1.png", # input_image
                     None, # end_image
                     0, # image_position
+                    1, # end_stillness
                     "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
                     "image", # generation_mode
                     "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
@@ -1945,6 +1957,7 @@ with block:
                     "./img_examples/Example2.webp", # input_image
                     None, # end_image
                     0, # image_position
+                    1, # end_stillness
                     "A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks, the man stops talking and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens",
                     "image", # generation_mode
                     "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
@@ -1969,6 +1982,7 @@ with block:
                     "./img_examples/Example2.webp", # input_image
                     None, # end_image
                     0, # image_position
+                    1, # end_stillness
                     "A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks, the woman stops talking and the woman listens A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens",
                     "image", # generation_mode
                     "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
@@ -1993,6 +2007,7 @@ with block:
                     "./img_examples/Example3.jpg", # input_image
                     None, # end_image
                     0, # image_position
+                    1, # end_stillness
                     "एउटा केटा दायाँतिर हिँडिरहेको छ, पूर्ण दृश्य, पूर्ण-लम्बाइको दृश्य, कार्टुन",
                     "image", # generation_mode
                     "हात छुटेको, लामो हात, अवास्तविक स्थिति, असम्भव विकृति, देखिने हड्डी, मांसपेशी संकुचन, कमजोर फ्रेम, धमिलो, धमिलो, अत्यधिक चिल्लो", # n_prompt
@@ -2017,6 +2032,7 @@ with block:
                     "./img_examples/Example4.webp", # input_image
                     None, # end_image
                     100, # image_position
+                    1, # end_stillness
                     "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
                     "image", # generation_mode
                     "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
@@ -2052,6 +2068,7 @@ with block:
                     "./img_examples/Example5.png", # input_image
                     "./img_examples/Example6.png", # end_image
                     0, # image_position
+                    1, # end_stillness
                     "A woman jumps out of the train and arrives on the ground, viewed from the outside, photorealistic, realistic, amateur photography, midday, insanely detailed, 8k", # prompt
                     "start_end", # generation_mode
                     "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth, jumpcut, crossfader, crossfading", # n_prompt
@@ -2206,7 +2223,7 @@ with block:
             gr.update(visible = False), # image_position
             gr.update(visible = True),  # input_image
             gr.update(visible = True),  # end_image
-            gr.update(visible = False), # end_stillness
+            gr.update(visible = True), # end_stillness
             gr.update(visible = False), # input_video
             gr.update(visible = True),  # start_button
             gr.update(visible = False), # start_button_video