Fabrice-TIERCELIN commited on
Commit
05baf72
·
verified ·
1 Parent(s): 34f6c24

Back before merge

Browse files
Files changed (1) hide show
  1. app.py +43 -469
app.py CHANGED
@@ -7,14 +7,7 @@ os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.di
7
  try:
8
  import spaces
9
  except:
10
- class spaces():
11
- def GPU(*args, **kwargs):
12
- def decorator(function):
13
- def new_function(*dummy_args, **dummy_kwargs):
14
- return function(*dummy_args, **dummy_kwargs)
15
- return new_function
16
- return decorator
17
-
18
  import gradio as gr
19
  import torch
20
  import traceback
@@ -121,7 +114,6 @@ os.makedirs(outputs_folder, exist_ok=True)
121
 
122
  input_image_debug_value = [None]
123
  input_video_debug_value = [None]
124
- end_image_debug_value = [None]
125
  prompt_debug_value = [None]
126
  total_second_length_debug_value = [None]
127
 
@@ -316,7 +308,7 @@ def set_mp4_comments_imageio_ffmpeg(input_file, comments):
316
  return False
317
 
318
  @torch.no_grad()
319
- def worker(input_image, image_position, end_image, prompts, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
320
  def encode_prompt(prompt, n_prompt):
321
  llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
322
 
@@ -585,275 +577,6 @@ def worker(input_image, image_position, end_image, prompts, n_prompt, seed, tota
585
  stream.output_queue.push(('end', None))
586
  return
587
 
588
- @torch.no_grad()
589
- def worker_start_end(input_image, image_position, end_image, prompts, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
590
- def encode_prompt(prompt, n_prompt):
591
- llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
592
-
593
- if cfg == 1:
594
- llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
595
- else:
596
- llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
597
-
598
- llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
599
- llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
600
-
601
- llama_vec = llama_vec.to(transformer.dtype)
602
- llama_vec_n = llama_vec_n.to(transformer.dtype)
603
- clip_l_pooler = clip_l_pooler.to(transformer.dtype)
604
- clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
605
- return [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n]
606
-
607
- total_latent_sections = (total_second_length * fps_number) / (latent_window_size * 4)
608
- total_latent_sections = int(max(round(total_latent_sections), 1))
609
-
610
- job_id = generate_timestamp()
611
-
612
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
613
-
614
- try:
615
- # Clean GPU
616
- if not high_vram:
617
- unload_complete_models(
618
- text_encoder, text_encoder_2, image_encoder, vae, transformer
619
- )
620
-
621
- # Text encoding
622
-
623
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
624
-
625
- if not high_vram:
626
- fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
627
- load_model_as_complete(text_encoder_2, target_device=gpu)
628
-
629
-
630
- prompt_parameters = []
631
-
632
- for prompt_part in prompts[:total_latent_sections]:
633
- prompt_parameters.append(encode_prompt(prompt_part, n_prompt))
634
-
635
- # Clean GPU
636
- if not high_vram:
637
- unload_complete_models(
638
- text_encoder, text_encoder_2
639
- )
640
-
641
- # Processing input image (start frame)
642
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Processing start frame ...'))))
643
-
644
- H, W, C = input_image.shape
645
- height, width = find_nearest_bucket(H, W, resolution=640)
646
- input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
647
-
648
- Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}_start.png'))
649
-
650
- input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
651
- input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
652
-
653
- # Processing end image (if provided)
654
- has_end_image = end_image is not None
655
- if has_end_image:
656
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Processing end frame ...'))))
657
-
658
- H_end, W_end, C_end = end_image.shape
659
- end_image_np = resize_and_center_crop(end_image, target_width=width, target_height=height)
660
-
661
- Image.fromarray(end_image_np).save(os.path.join(outputs_folder, f'{job_id}_end.png'))
662
-
663
- end_image_pt = torch.from_numpy(end_image_np).float() / 127.5 - 1
664
- end_image_pt = end_image_pt.permute(2, 0, 1)[None, :, None]
665
-
666
- # VAE encoding
667
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
668
-
669
- if not high_vram:
670
- load_model_as_complete(vae, target_device=gpu)
671
-
672
- start_latent = vae_encode(input_image_pt, vae)
673
-
674
- if has_end_image:
675
- end_latent = vae_encode(end_image_pt, vae)
676
-
677
- # CLIP Vision
678
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
679
-
680
- if not high_vram:
681
- load_model_as_complete(image_encoder, target_device=gpu)
682
-
683
- image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
684
- image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
685
-
686
- if has_end_image:
687
- end_image_encoder_output = hf_clip_vision_encode(end_image_np, feature_extractor, image_encoder)
688
- end_image_encoder_last_hidden_state = end_image_encoder_output.last_hidden_state
689
- # Combine both image embeddings or use a weighted approach
690
- image_encoder_last_hidden_state = (image_encoder_last_hidden_state + end_image_encoder_last_hidden_state) / 2
691
-
692
- # Clean GPU
693
- if not high_vram:
694
- unload_complete_models(
695
- image_encoder
696
- )
697
-
698
- # Dtype
699
- image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
700
-
701
- # Sampling
702
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
703
-
704
- rnd = torch.Generator("cpu").manual_seed(seed)
705
- num_frames = latent_window_size * 4 - 3
706
-
707
- history_latents = torch.zeros(size=(1, 16, 1 + 2 + 16, height // 8, width // 8), dtype=torch.float32, device=cpu)
708
- start_latent = start_latent.to(history_latents)
709
- if has_end_image:
710
- end_latent = end_latent.to(history_latents)
711
-
712
- history_pixels = None
713
- total_generated_latent_frames = 0
714
-
715
- if total_latent_sections > 4:
716
- # In theory the latent_paddings should follow the above sequence, but it seems that duplicating some
717
- # items looks better than expanding it when total_latent_sections > 4
718
- # One can try to remove below trick and just
719
- # use `latent_paddings = list(reversed(range(total_latent_sections)))` to compare
720
- latent_paddings = [3] + [2] * (total_latent_sections - 3) + [1, 0]
721
- else:
722
- # Convert an iterator to a list
723
- latent_paddings = list(range(total_latent_sections - 1, -1, -1))
724
-
725
- if enable_preview:
726
- def callback(d):
727
- preview = d['denoised']
728
- preview = vae_decode_fake(preview)
729
-
730
- preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
731
- preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
732
-
733
- if stream.input_queue.top() == 'end':
734
- stream.output_queue.push(('end', None))
735
- raise KeyboardInterrupt('User ends the task.')
736
-
737
- current_step = d['i'] + 1
738
- percentage = int(100.0 * current_step / steps)
739
- hint = f'Sampling {current_step}/{steps}'
740
- desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / fps_number) :.2f} seconds (FPS-30), Resolution: {height}px * {width}px. The video is being extended now ...'
741
- stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
742
- return
743
- else:
744
- def callback(d):
745
- return
746
-
747
- for latent_padding in latent_paddings:
748
- is_last_section = latent_padding == 0
749
- is_first_section = latent_padding == latent_paddings[0]
750
- latent_padding_size = latent_padding * latent_window_size
751
-
752
- if stream.input_queue.top() == 'end':
753
- stream.output_queue.push(('end', None))
754
- return
755
-
756
- print(f'latent_padding_size = {latent_padding_size}, is_last_section = {is_last_section}, is_first_section = {is_first_section}')
757
-
758
- if len(prompt_parameters) > 0:
759
- [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(len(prompt_parameters) - 1)
760
-
761
- indices = torch.arange(1 + latent_padding_size + latent_window_size + 1 + 2 + 16).unsqueeze(0)
762
- clean_latent_indices_pre, blank_indices, latent_indices, clean_latent_indices_post, clean_latent_2x_indices, clean_latent_4x_indices = indices.split([1, latent_padding_size, latent_window_size, 1, 2, 16], dim=1)
763
- clean_latent_indices = torch.cat([clean_latent_indices_pre, clean_latent_indices_post], dim=1)
764
-
765
- clean_latents_post, clean_latents_2x, clean_latents_4x = history_latents[:, :, :1 + 2 + 16, :, :].split([1, 2, 16], dim=2)
766
-
767
- # Use end image latent for the first section if provided
768
- if has_end_image and is_first_section:
769
- clean_latents_post = end_latent
770
-
771
- clean_latents = torch.cat([start_latent, clean_latents_post], dim=2)
772
-
773
- if not high_vram:
774
- unload_complete_models()
775
- move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
776
-
777
- if use_teacache:
778
- transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
779
- else:
780
- transformer.initialize_teacache(enable_teacache=False)
781
-
782
- generated_latents = sample_hunyuan(
783
- transformer=transformer,
784
- sampler='unipc',
785
- width=width,
786
- height=height,
787
- frames=num_frames,
788
- real_guidance_scale=cfg,
789
- distilled_guidance_scale=gs,
790
- guidance_rescale=rs,
791
- # shift=3.0,
792
- num_inference_steps=steps,
793
- generator=rnd,
794
- prompt_embeds=llama_vec,
795
- prompt_embeds_mask=llama_attention_mask,
796
- prompt_poolers=clip_l_pooler,
797
- negative_prompt_embeds=llama_vec_n,
798
- negative_prompt_embeds_mask=llama_attention_mask_n,
799
- negative_prompt_poolers=clip_l_pooler_n,
800
- device=gpu,
801
- dtype=torch.bfloat16,
802
- image_embeddings=image_encoder_last_hidden_state,
803
- latent_indices=latent_indices,
804
- clean_latents=clean_latents,
805
- clean_latent_indices=clean_latent_indices,
806
- clean_latents_2x=clean_latents_2x,
807
- clean_latent_2x_indices=clean_latent_2x_indices,
808
- clean_latents_4x=clean_latents_4x,
809
- clean_latent_4x_indices=clean_latent_4x_indices,
810
- callback=callback,
811
- )
812
-
813
- if is_last_section:
814
- generated_latents = torch.cat([start_latent.to(generated_latents), generated_latents], dim=2)
815
-
816
- total_generated_latent_frames += int(generated_latents.shape[2])
817
- history_latents = torch.cat([generated_latents.to(history_latents), history_latents], dim=2)
818
-
819
- if not high_vram:
820
- offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
821
- load_model_as_complete(vae, target_device=gpu)
822
-
823
- if history_pixels is None:
824
- history_pixels = vae_decode(history_latents[:, :, :total_generated_latent_frames, :, :], vae).cpu()
825
- else:
826
- section_latent_frames = (latent_window_size * 2 + 1) if is_last_section else (latent_window_size * 2)
827
- overlapped_frames = latent_window_size * 4 - 3
828
-
829
- current_pixels = vae_decode(history_latents[:, :, :min(total_generated_latent_frames, section_latent_frames)], vae).cpu()
830
- history_pixels = soft_append_bcthw(current_pixels, history_pixels, overlapped_frames)
831
-
832
- if not high_vram:
833
- unload_complete_models(vae)
834
-
835
- if enable_preview or is_last_section:
836
- output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
837
-
838
- save_bcthw_as_mp4(history_pixels, output_filename, fps=fps_number, crf=mp4_crf)
839
-
840
- print(f'Decoded. Pixel shape {history_pixels.shape}')
841
-
842
- stream.output_queue.push(('file', output_filename))
843
-
844
- if is_last_section:
845
- break
846
- except:
847
- traceback.print_exc()
848
-
849
- if not high_vram:
850
- unload_complete_models(
851
- text_encoder, text_encoder_2, image_encoder, vae, transformer
852
- )
853
-
854
- stream.output_queue.push(('end', None))
855
- return
856
-
857
  # 20250506 pftq: Modified worker to accept video input and clean frame count
858
  @torch.no_grad()
859
  def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
@@ -1134,17 +857,18 @@ def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_
1134
  stream.output_queue.push(('end', None))
1135
  return
1136
 
1137
- def get_duration(input_image, image_position, end_image, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
1138
  return allocation_time
1139
 
 
1140
  @spaces.GPU(duration=get_duration)
1141
- def process_on_gpu(input_image, image_position, end_image, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number
1142
  ):
1143
  start = time.time()
1144
  global stream
1145
  stream = AsyncStream()
1146
 
1147
- async_run(worker_start_end if generation_mode == "start_end" else worker, input_image, image_position, end_image, prompts, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number)
1148
 
1149
  output_filename = None
1150
 
@@ -1175,7 +899,6 @@ def process_on_gpu(input_image, image_position, end_image, prompts, generation_m
1175
 
1176
  def process(input_image,
1177
  image_position=0,
1178
- end_image=None,
1179
  prompt="",
1180
  generation_mode="image",
1181
  n_prompt="",
@@ -1186,12 +909,12 @@ def process(input_image,
1186
  resolution=640,
1187
  total_second_length=5,
1188
  latent_window_size=9,
1189
- steps=30,
1190
  cfg=1.0,
1191
  gs=10.0,
1192
  rs=0.0,
1193
  gpu_memory_preservation=6,
1194
- enable_preview=False,
1195
  use_teacache=False,
1196
  mp4_crf=16,
1197
  fps_number=30
@@ -1199,13 +922,12 @@ def process(input_image,
1199
  if auto_allocation:
1200
  allocation_time = min(total_second_length * 60 * (1.5 if use_teacache else 3.0) * (1 + ((steps - 25) / 25))**2, 600)
1201
 
1202
- if input_image_debug_value[0] is not None or end_image_debug_value[0] is not None or prompt_debug_value[0] is not None or total_second_length_debug_value[0] is not None:
1203
  input_image = input_image_debug_value[0]
1204
- end_image = end_image_debug_value[0]
1205
  prompt = prompt_debug_value[0]
1206
  total_second_length = total_second_length_debug_value[0]
1207
  allocation_time = min(total_second_length_debug_value[0] * 60 * 100, 600)
1208
- input_image_debug_value[0] = end_image_debug_value[0] = input_video_debug_value[0] = prompt_debug_value[0] = total_second_length_debug_value[0] = None
1209
 
1210
  if torch.cuda.device_count() == 0:
1211
  gr.Warning('Set this space to GPU config to make it work.')
@@ -1227,7 +949,6 @@ def process(input_image,
1227
 
1228
  yield from process_on_gpu(input_image,
1229
  image_position,
1230
- end_image,
1231
  prompts,
1232
  generation_mode,
1233
  n_prompt,
@@ -1250,6 +971,7 @@ def process(input_image,
1250
  def get_duration_video(input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
1251
  return allocation_time
1252
 
 
1253
  @spaces.GPU(duration=get_duration_video)
1254
  def process_video_on_gpu(input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
1255
  start = time.time()
@@ -1297,7 +1019,7 @@ def process_video(input_video, prompt, n_prompt, randomize_seed, seed, auto_allo
1297
  prompt = prompt_debug_value[0]
1298
  total_second_length = total_second_length_debug_value[0]
1299
  allocation_time = min(total_second_length_debug_value[0] * 60 * 100, 600)
1300
- input_image_debug_value[0] = end_image_debug_value[0] = input_video_debug_value[0] = prompt_debug_value[0] = total_second_length_debug_value[0] = None
1301
 
1302
  if torch.cuda.device_count() == 0:
1303
  gr.Warning('Set this space to GPU config to make it work.')
@@ -1397,10 +1119,9 @@ with block:
1397
  local_storage = gr.BrowserState(default_local_storage)
1398
  with gr.Row():
1399
  with gr.Column():
1400
- generation_mode = gr.Radio([["Text-to-Video", "text"], ["Image-to-Video", "image"], ["Start & end frames", "start_end"], ["Video Extension", "video"]], elem_id="generation-mode", label="Generation mode", value = "image")
1401
  text_to_video_hint = gr.HTML("Text-to-Video badly works with a flash effect at the start. I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
1402
  input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
1403
- end_image = gr.Image(sources='upload', type="numpy", label="End Frame (Optional)", height=320)
1404
  image_position = gr.Slider(label="Image position", minimum=0, maximum=100, value=0, step=1, info='0=Video start; 100=Video end (lower quality)')
1405
  input_video = gr.Video(sources='upload', label="Input Video", height=320)
1406
  timeless_prompt = gr.Textbox(label="Timeless prompt", info='Used on the whole duration of the generation', value='', placeholder="The creature starts to move, fast motion, fixed camera, focus motion, consistent arm, consistent position, mute colors, insanely detailed")
@@ -1426,7 +1147,7 @@ with block:
1426
  enable_preview = gr.Checkbox(label='Enable preview', value=True, info='Display a preview around each second generated but it costs 2 sec. for each second generated.')
1427
  use_teacache = gr.Checkbox(label='Use TeaCache', value=False, info='Faster speed and no break in brightness, but often makes hands and fingers slightly worse.')
1428
 
1429
- n_prompt = gr.Textbox(label="Negative Prompt", value="Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", info='Requires using normal CFG (undistilled) instead of Distilled (set Distilled=1 and CFG > 1).')
1430
 
1431
  fps_number = gr.Slider(label="Frame per seconds", info="The model is trained for 30 fps so other fps may generate weird results", minimum=10, maximum=60, value=30, step=1)
1432
 
@@ -1476,7 +1197,6 @@ with block:
1476
 
1477
  with gr.Accordion("Debug", open=False):
1478
  input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320)
1479
- end_image_debug = gr.Image(type="numpy", label="End Image Debug", height=320)
1480
  input_video_debug = gr.Video(sources='upload', label="Input Video Debug", height=320)
1481
  prompt_debug = gr.Textbox(label="Prompt Debug", value='')
1482
  total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (seconds) Debug", minimum=1, maximum=120, value=1, step=0.1)
@@ -1488,7 +1208,8 @@ with block:
1488
  progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
1489
  progress_bar = gr.HTML('', elem_classes='no-generating-animation')
1490
 
1491
- ips = [input_image, image_position, end_image, final_prompt, generation_mode, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number]
 
1492
  ips_video = [input_video, final_prompt, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch]
1493
 
1494
  with gr.Row(elem_id="text_examples", visible=False):
@@ -1498,10 +1219,9 @@ with block:
1498
  [
1499
  None, # input_image
1500
  0, # image_position
1501
- None, # end_image
1502
  "Overcrowed street in Japan, photorealistic, realistic, intricate details, 8k, insanely detailed",
1503
  "text", # generation_mode
1504
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1505
  True, # randomize_seed
1506
  42, # seed
1507
  True, # auto_allocation
@@ -1534,10 +1254,9 @@ with block:
1534
  [
1535
  "./img_examples/Example2.webp", # input_image
1536
  0, # image_position
1537
- None, # end_image
1538
  "A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks, the man stops talking and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens",
1539
  "image", # generation_mode
1540
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1541
  True, # randomize_seed
1542
  42, # seed
1543
  True, # auto_allocation
@@ -1558,10 +1277,9 @@ with block:
1558
  [
1559
  "./img_examples/Example1.png", # input_image
1560
  0, # image_position
1561
- None, # end_image
1562
  "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
1563
  "image", # generation_mode
1564
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1565
  True, # randomize_seed
1566
  42, # seed
1567
  True, # auto_allocation
@@ -1582,10 +1300,9 @@ with block:
1582
  [
1583
  "./img_examples/Example4.webp", # input_image
1584
  1, # image_position
1585
- None, # end_image
1586
  "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
1587
  "image", # generation_mode
1588
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1589
  True, # randomize_seed
1590
  42, # seed
1591
  True, # auto_allocation
@@ -1606,10 +1323,9 @@ with block:
1606
  [
1607
  "./img_examples/Example4.webp", # input_image
1608
  50, # image_position
1609
- None, # end_image
1610
  "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
1611
  "image", # generation_mode
1612
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1613
  True, # randomize_seed
1614
  42, # seed
1615
  True, # auto_allocation
@@ -1630,46 +1346,9 @@ with block:
1630
  [
1631
  "./img_examples/Example4.webp", # input_image
1632
  100, # image_position
1633
- None, # end_image
1634
  "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
1635
  "image", # generation_mode
1636
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1637
- True, # randomize_seed
1638
- 42, # seed
1639
- True, # auto_allocation
1640
- 180, # allocation_time
1641
- 672, # resolution
1642
- 1, # total_second_length
1643
- 9, # latent_window_size
1644
- 30, # steps
1645
- 1.0, # cfg
1646
- 10.0, # gs
1647
- 0.0, # rs
1648
- 6, # gpu_memory_preservation
1649
- False, # enable_preview
1650
- False, # use_teacache
1651
- 16, # mp4_crf
1652
- 30 # fps_number
1653
- ],
1654
- ],
1655
- run_on_click = True,
1656
- fn = process,
1657
- inputs = ips,
1658
- outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button, warning],
1659
- cache_examples = torch.cuda.device_count() > 0,
1660
- )
1661
-
1662
- with gr.Row(elem_id="start_end_examples", visible=False):
1663
- gr.Examples(
1664
- label = "Examples from start and end frames",
1665
- examples = [
1666
- [
1667
- "./img_examples/Example2.webp", # input_image
1668
- 0, # image_position
1669
- None, # end_image
1670
- "A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks, the man stops talking and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens",
1671
- "start_end", # generation_mode
1672
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1673
  True, # randomize_seed
1674
  42, # seed
1675
  True, # auto_allocation
@@ -1702,7 +1381,7 @@ with block:
1702
  [
1703
  "./img_examples/Example1.mp4", # input_video
1704
  "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
1705
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1706
  True, # randomize_seed
1707
  42, # seed
1708
  True, # auto_allocation
@@ -1726,7 +1405,7 @@ with block:
1726
  [
1727
  "./img_examples/Example1.mp4", # input_video
1728
  "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
1729
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1730
  True, # randomize_seed
1731
  42, # seed
1732
  True, # auto_allocation
@@ -1761,10 +1440,9 @@ with block:
1761
  [
1762
  None, # input_image
1763
  0, # image_position
1764
- None, # end_image
1765
  "Overcrowed street in Japan, photorealistic, realistic, intricate details, 8k, insanely detailed",
1766
  "text", # generation_mode
1767
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1768
  True, # randomize_seed
1769
  42, # seed
1770
  True, # auto_allocation
@@ -1796,10 +1474,9 @@ with block:
1796
  [
1797
  "./img_examples/Example1.png", # input_image
1798
  0, # image_position
1799
- None, # end_image
1800
  "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
1801
  "image", # generation_mode
1802
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1803
  True, # randomize_seed
1804
  42, # seed
1805
  True, # auto_allocation
@@ -1820,10 +1497,9 @@ with block:
1820
  [
1821
  "./img_examples/Example2.webp", # input_image
1822
  0, # image_position
1823
- None, # end_image
1824
  "A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks, the man stops talking and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens",
1825
  "image", # generation_mode
1826
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1827
  True, # randomize_seed
1828
  42, # seed
1829
  True, # auto_allocation
@@ -1844,10 +1520,9 @@ with block:
1844
  [
1845
  "./img_examples/Example2.webp", # input_image
1846
  0, # image_position
1847
- None, # end_image
1848
  "A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks, the woman stops talking and the woman listens A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens",
1849
  "image", # generation_mode
1850
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1851
  True, # randomize_seed
1852
  42, # seed
1853
  True, # auto_allocation
@@ -1868,10 +1543,9 @@ with block:
1868
  [
1869
  "./img_examples/Example3.jpg", # input_image
1870
  0, # image_position
1871
- None, # end_image
1872
  "A boy is walking to the right, full view, full-length view, cartoon",
1873
  "image", # generation_mode
1874
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1875
  True, # randomize_seed
1876
  42, # seed
1877
  True, # auto_allocation
@@ -1892,10 +1566,9 @@ with block:
1892
  [
1893
  "./img_examples/Example4.webp", # input_image
1894
  100, # image_position
1895
- None, # end_image
1896
  "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
1897
  "image", # generation_mode
1898
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1899
  True, # randomize_seed
1900
  42, # seed
1901
  True, # auto_allocation
@@ -1921,48 +1594,13 @@ with block:
1921
  cache_examples = False,
1922
  )
1923
 
1924
- gr.Examples(
1925
- label = "🖼️ Examples from start and end frames",
1926
- examples = [
1927
- [
1928
- "./img_examples/Example1.png", # input_image
1929
- 0, # image_position
1930
- None, # end_image
1931
- "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
1932
- "start_end", # generation_mode
1933
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1934
- True, # randomize_seed
1935
- 42, # seed
1936
- True, # auto_allocation
1937
- 180, # allocation_time
1938
- 672, # resolution
1939
- 1, # total_second_length
1940
- 9, # latent_window_size
1941
- 30, # steps
1942
- 1.0, # cfg
1943
- 10.0, # gs
1944
- 0.0, # rs
1945
- 6, # gpu_memory_preservation
1946
- False, # enable_preview
1947
- True, # use_teacache
1948
- 16, # mp4_crf
1949
- 30 # fps_number
1950
- ],
1951
- ],
1952
- run_on_click = True,
1953
- fn = process,
1954
- inputs = ips,
1955
- outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button, warning],
1956
- cache_examples = False,
1957
- )
1958
-
1959
  gr.Examples(
1960
  label = "🎥 Examples from video",
1961
  examples = [
1962
  [
1963
  "./img_examples/Example1.mp4", # input_video
1964
  "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
1965
- "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, poorly framed, blurred, blurry, over-smooth", # n_prompt
1966
  True, # randomize_seed
1967
  42, # seed
1968
  True, # auto_allocation
@@ -2013,106 +1651,42 @@ with block:
2013
 
2014
  def handle_generation_mode_change(generation_mode_data):
2015
  if generation_mode_data == "text":
2016
- return [
2017
- gr.update(visible = True), # text_to_video_hint
2018
- gr.update(visible = False), # image_position
2019
- gr.update(visible = False), # input_image
2020
- gr.update(visible = False), # end_image
2021
- gr.update(visible = False), # input_video
2022
- gr.update(visible = True), # start_button
2023
- gr.update(visible = False), # start_button_video
2024
- gr.update(visible = False), # no_resize
2025
- gr.update(visible = False), # batch
2026
- gr.update(visible = False), # num_clean_frames
2027
- gr.update(visible = False), # vae_batch
2028
- gr.update(visible = False), # prompt_hint
2029
- gr.update(visible = True) # fps_number
2030
- ]
2031
  elif generation_mode_data == "image":
2032
- return [
2033
- gr.update(visible = False), # text_to_video_hint
2034
- gr.update(visible = True), # image_position
2035
- gr.update(visible = True), # input_image
2036
- gr.update(visible = False), # end_image
2037
- gr.update(visible = False), # input_video
2038
- gr.update(visible = True), # start_button
2039
- gr.update(visible = False), # start_button_video
2040
- gr.update(visible = False), # no_resize
2041
- gr.update(visible = False), # batch
2042
- gr.update(visible = False), # num_clean_frames
2043
- gr.update(visible = False), # vae_batch
2044
- gr.update(visible = False), # prompt_hint
2045
- gr.update(visible = True) # fps_number
2046
- ]
2047
- elif generation_mode_data == "start_end":
2048
- return [
2049
- gr.update(visible = False), # text_to_video_hint
2050
- gr.update(visible = False), # image_position
2051
- gr.update(visible = True), # input_image
2052
- gr.update(visible = True), # end_image
2053
- gr.update(visible = False), # input_video
2054
- gr.update(visible = True), # start_button
2055
- gr.update(visible = False), # start_button_video
2056
- gr.update(visible = False), # no_resize
2057
- gr.update(visible = False), # batch
2058
- gr.update(visible = False), # num_clean_frames
2059
- gr.update(visible = False), # vae_batch
2060
- gr.update(visible = False), # prompt_hint
2061
- gr.update(visible = True) # fps_number
2062
- ]
2063
  elif generation_mode_data == "video":
2064
- return [
2065
- gr.update(visible = False), # text_to_video_hint
2066
- gr.update(visible = False), # image_position
2067
- gr.update(visible = False), # input_image
2068
- gr.update(visible = False), # end_image
2069
- gr.update(visible = True), # input_video
2070
- gr.update(visible = False), # start_button
2071
- gr.update(visible = True), # start_button_video
2072
- gr.update(visible = True), # no_resize
2073
- gr.update(visible = True), # batch
2074
- gr.update(visible = True), # num_clean_frames
2075
- gr.update(visible = True), # vae_batch
2076
- gr.update(visible = True), # prompt_hint
2077
- gr.update(visible = False) # fps_number
2078
- ]
2079
-
2080
- def handle_field_debug_change(input_image_debug_data, input_video_debug_data, end_image_debug_data, prompt_debug_data, total_second_length_debug_data):
2081
  print("handle_field_debug_change")
2082
  input_image_debug_value[0] = input_image_debug_data
2083
  input_video_debug_value[0] = input_video_debug_data
2084
- end_image_debug_value[0] = end_image_debug_data
2085
  prompt_debug_value[0] = prompt_debug_data
2086
  total_second_length_debug_value[0] = total_second_length_debug_data
2087
  return []
2088
 
2089
  input_image_debug.upload(
2090
  fn=handle_field_debug_change,
2091
- inputs=[input_image_debug, input_video_debug, end_image_debug, prompt_debug, total_second_length_debug],
2092
  outputs=[]
2093
  )
2094
 
2095
  input_video_debug.upload(
2096
  fn=handle_field_debug_change,
2097
- inputs=[input_image_debug, input_video_debug, end_image_debug, prompt_debug, total_second_length_debug],
2098
- outputs=[]
2099
- )
2100
-
2101
- end_image_debug.upload(
2102
- fn=handle_field_debug_change,
2103
- inputs=[input_image_debug, input_video_debug, end_image_debug, prompt_debug, total_second_length_debug],
2104
  outputs=[]
2105
  )
2106
 
2107
  prompt_debug.change(
2108
  fn=handle_field_debug_change,
2109
- inputs=[input_image_debug, input_video_debug, end_image_debug, prompt_debug, total_second_length_debug],
2110
  outputs=[]
2111
  )
2112
 
2113
  total_second_length_debug.change(
2114
  fn=handle_field_debug_change,
2115
- inputs=[input_image_debug, input_video_debug, end_image_debug, prompt_debug, total_second_length_debug],
2116
  outputs=[]
2117
  )
2118
 
@@ -2136,7 +1710,7 @@ with block:
2136
  generation_mode.change(
2137
  fn=handle_generation_mode_change,
2138
  inputs=[generation_mode],
2139
- outputs=[text_to_video_hint, image_position, input_image, end_image, input_video, start_button, start_button_video, no_resize, batch, num_clean_frames, vae_batch, prompt_hint, fps_number]
2140
  )
2141
 
2142
  # Update display when the page loads
@@ -2144,7 +1718,7 @@ with block:
2144
  fn=handle_generation_mode_change, inputs = [
2145
  generation_mode
2146
  ], outputs = [
2147
- text_to_video_hint, image_position, input_image, end_image, input_video, start_button, start_button_video, no_resize, batch, num_clean_frames, vae_batch, prompt_hint, fps_number
2148
  ]
2149
  )
2150
 
 
7
  try:
8
  import spaces
9
  except:
10
+ print("Not on HuggingFace")
 
 
 
 
 
 
 
11
  import gradio as gr
12
  import torch
13
  import traceback
 
114
 
115
  input_image_debug_value = [None]
116
  input_video_debug_value = [None]
 
117
  prompt_debug_value = [None]
118
  total_second_length_debug_value = [None]
119
 
 
308
  return False
309
 
310
  @torch.no_grad()
311
+ def worker(input_image, image_position, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
312
  def encode_prompt(prompt, n_prompt):
313
  llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
314
 
 
577
  stream.output_queue.push(('end', None))
578
  return
579
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
580
  # 20250506 pftq: Modified worker to accept video input and clean frame count
581
  @torch.no_grad()
582
  def worker_video(input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
 
857
  stream.output_queue.push(('end', None))
858
  return
859
 
860
+ def get_duration(input_image, image_position, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number):
861
  return allocation_time
862
 
863
+ # Remove this decorator if you run on local
864
  @spaces.GPU(duration=get_duration)
865
+ def process_on_gpu(input_image, image_position, prompts, generation_mode, n_prompt, seed, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number
866
  ):
867
  start = time.time()
868
  global stream
869
  stream = AsyncStream()
870
 
871
+ async_run(worker, input_image, image_position, prompts, n_prompt, seed, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number)
872
 
873
  output_filename = None
874
 
 
899
 
900
  def process(input_image,
901
  image_position=0,
 
902
  prompt="",
903
  generation_mode="image",
904
  n_prompt="",
 
909
  resolution=640,
910
  total_second_length=5,
911
  latent_window_size=9,
912
+ steps=25,
913
  cfg=1.0,
914
  gs=10.0,
915
  rs=0.0,
916
  gpu_memory_preservation=6,
917
+ enable_preview=True,
918
  use_teacache=False,
919
  mp4_crf=16,
920
  fps_number=30
 
922
  if auto_allocation:
923
  allocation_time = min(total_second_length * 60 * (1.5 if use_teacache else 3.0) * (1 + ((steps - 25) / 25))**2, 600)
924
 
925
+ if input_image_debug_value[0] is not None or prompt_debug_value[0] is not None or total_second_length_debug_value[0] is not None:
926
  input_image = input_image_debug_value[0]
 
927
  prompt = prompt_debug_value[0]
928
  total_second_length = total_second_length_debug_value[0]
929
  allocation_time = min(total_second_length_debug_value[0] * 60 * 100, 600)
930
+ input_image_debug_value[0] = prompt_debug_value[0] = total_second_length_debug_value[0] = None
931
 
932
  if torch.cuda.device_count() == 0:
933
  gr.Warning('Set this space to GPU config to make it work.')
 
949
 
950
  yield from process_on_gpu(input_image,
951
  image_position,
 
952
  prompts,
953
  generation_mode,
954
  n_prompt,
 
971
  def get_duration_video(input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
972
  return allocation_time
973
 
974
+ # Remove this decorator if you run on local
975
  @spaces.GPU(duration=get_duration_video)
976
  def process_video_on_gpu(input_video, prompts, n_prompt, seed, batch, resolution, total_second_length, allocation_time, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch):
977
  start = time.time()
 
1019
  prompt = prompt_debug_value[0]
1020
  total_second_length = total_second_length_debug_value[0]
1021
  allocation_time = min(total_second_length_debug_value[0] * 60 * 100, 600)
1022
+ input_video_debug_value[0] = prompt_debug_value[0] = total_second_length_debug_value[0] = None
1023
 
1024
  if torch.cuda.device_count() == 0:
1025
  gr.Warning('Set this space to GPU config to make it work.')
 
1119
  local_storage = gr.BrowserState(default_local_storage)
1120
  with gr.Row():
1121
  with gr.Column():
1122
+ generation_mode = gr.Radio([["Text-to-Video", "text"], ["Image-to-Video", "image"], ["Video Extension", "video"]], elem_id="generation-mode", label="Generation mode", value = "image")
1123
  text_to_video_hint = gr.HTML("Text-to-Video badly works with a flash effect at the start. I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
1124
  input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
 
1125
  image_position = gr.Slider(label="Image position", minimum=0, maximum=100, value=0, step=1, info='0=Video start; 100=Video end (lower quality)')
1126
  input_video = gr.Video(sources='upload', label="Input Video", height=320)
1127
  timeless_prompt = gr.Textbox(label="Timeless prompt", info='Used on the whole duration of the generation', value='', placeholder="The creature starts to move, fast motion, fixed camera, focus motion, consistent arm, consistent position, mute colors, insanely detailed")
 
1147
  enable_preview = gr.Checkbox(label='Enable preview', value=True, info='Display a preview around each second generated but it costs 2 sec. for each second generated.')
1148
  use_teacache = gr.Checkbox(label='Use TeaCache', value=False, info='Faster speed and no break in brightness, but often makes hands and fingers slightly worse.')
1149
 
1150
+ n_prompt = gr.Textbox(label="Negative Prompt", value="Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", info='Requires using normal CFG (undistilled) instead of Distilled (set Distilled=1 and CFG > 1).')
1151
 
1152
  fps_number = gr.Slider(label="Frame per seconds", info="The model is trained for 30 fps so other fps may generate weird results", minimum=10, maximum=60, value=30, step=1)
1153
 
 
1197
 
1198
  with gr.Accordion("Debug", open=False):
1199
  input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320)
 
1200
  input_video_debug = gr.Video(sources='upload', label="Input Video Debug", height=320)
1201
  prompt_debug = gr.Textbox(label="Prompt Debug", value='')
1202
  total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (seconds) Debug", minimum=1, maximum=120, value=1, step=0.1)
 
1208
  progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
1209
  progress_bar = gr.HTML('', elem_classes='no-generating-animation')
1210
 
1211
+ # 20250506 pftq: Updated inputs to include num_clean_frames
1212
+ ips = [input_image, image_position, final_prompt, generation_mode, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, mp4_crf, fps_number]
1213
  ips_video = [input_video, final_prompt, n_prompt, randomize_seed, seed, auto_allocation, allocation_time, batch, resolution, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, enable_preview, use_teacache, no_resize, mp4_crf, num_clean_frames, vae_batch]
1214
 
1215
  with gr.Row(elem_id="text_examples", visible=False):
 
1219
  [
1220
  None, # input_image
1221
  0, # image_position
 
1222
  "Overcrowed street in Japan, photorealistic, realistic, intricate details, 8k, insanely detailed",
1223
  "text", # generation_mode
1224
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1225
  True, # randomize_seed
1226
  42, # seed
1227
  True, # auto_allocation
 
1254
  [
1255
  "./img_examples/Example2.webp", # input_image
1256
  0, # image_position
 
1257
  "A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks, the man stops talking and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens",
1258
  "image", # generation_mode
1259
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1260
  True, # randomize_seed
1261
  42, # seed
1262
  True, # auto_allocation
 
1277
  [
1278
  "./img_examples/Example1.png", # input_image
1279
  0, # image_position
 
1280
  "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
1281
  "image", # generation_mode
1282
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1283
  True, # randomize_seed
1284
  42, # seed
1285
  True, # auto_allocation
 
1300
  [
1301
  "./img_examples/Example4.webp", # input_image
1302
  1, # image_position
 
1303
  "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
1304
  "image", # generation_mode
1305
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1306
  True, # randomize_seed
1307
  42, # seed
1308
  True, # auto_allocation
 
1323
  [
1324
  "./img_examples/Example4.webp", # input_image
1325
  50, # image_position
 
1326
  "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
1327
  "image", # generation_mode
1328
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1329
  True, # randomize_seed
1330
  42, # seed
1331
  True, # auto_allocation
 
1346
  [
1347
  "./img_examples/Example4.webp", # input_image
1348
  100, # image_position
 
1349
  "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
1350
  "image", # generation_mode
1351
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1352
  True, # randomize_seed
1353
  42, # seed
1354
  True, # auto_allocation
 
1381
  [
1382
  "./img_examples/Example1.mp4", # input_video
1383
  "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
1384
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1385
  True, # randomize_seed
1386
  42, # seed
1387
  True, # auto_allocation
 
1405
  [
1406
  "./img_examples/Example1.mp4", # input_video
1407
  "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
1408
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1409
  True, # randomize_seed
1410
  42, # seed
1411
  True, # auto_allocation
 
1440
  [
1441
  None, # input_image
1442
  0, # image_position
 
1443
  "Overcrowed street in Japan, photorealistic, realistic, intricate details, 8k, insanely detailed",
1444
  "text", # generation_mode
1445
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1446
  True, # randomize_seed
1447
  42, # seed
1448
  True, # auto_allocation
 
1474
  [
1475
  "./img_examples/Example1.png", # input_image
1476
  0, # image_position
 
1477
  "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
1478
  "image", # generation_mode
1479
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1480
  True, # randomize_seed
1481
  42, # seed
1482
  True, # auto_allocation
 
1497
  [
1498
  "./img_examples/Example2.webp", # input_image
1499
  0, # image_position
 
1500
  "A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks, the man stops talking and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens",
1501
  "image", # generation_mode
1502
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1503
  True, # randomize_seed
1504
  42, # seed
1505
  True, # auto_allocation
 
1520
  [
1521
  "./img_examples/Example2.webp", # input_image
1522
  0, # image_position
 
1523
  "A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The woman talks and the man listens; A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks, the woman stops talking and the woman listens A man on the left and a woman on the right face each other ready to start a conversation, large space between the persons, full view, full-length view, 3D, pixar, 3D render, CGI. The man talks and the woman listens",
1524
  "image", # generation_mode
1525
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1526
  True, # randomize_seed
1527
  42, # seed
1528
  True, # auto_allocation
 
1543
  [
1544
  "./img_examples/Example3.jpg", # input_image
1545
  0, # image_position
 
1546
  "A boy is walking to the right, full view, full-length view, cartoon",
1547
  "image", # generation_mode
1548
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1549
  True, # randomize_seed
1550
  42, # seed
1551
  True, # auto_allocation
 
1566
  [
1567
  "./img_examples/Example4.webp", # input_image
1568
  100, # image_position
 
1569
  "A building starting to explode, photorealistic, realisitc, 8k, insanely detailed",
1570
  "image", # generation_mode
1571
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1572
  True, # randomize_seed
1573
  42, # seed
1574
  True, # auto_allocation
 
1594
  cache_examples = False,
1595
  )
1596
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1597
  gr.Examples(
1598
  label = "🎥 Examples from video",
1599
  examples = [
1600
  [
1601
  "./img_examples/Example1.mp4", # input_video
1602
  "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
1603
+ "Missing arm, long hand, unrealistic position, impossible contortion, visible bone, muscle contraction, blurred, blurry, over-smooth", # n_prompt
1604
  True, # randomize_seed
1605
  42, # seed
1606
  True, # auto_allocation
 
1651
 
1652
  def handle_generation_mode_change(generation_mode_data):
1653
  if generation_mode_data == "text":
1654
+ return [gr.update(visible = True), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = True), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = True)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1655
  elif generation_mode_data == "image":
1656
+ return [gr.update(visible = False), gr.update(visible = True), gr.update(visible = True), gr.update(visible = False), gr.update(visible = True), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = True)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1657
  elif generation_mode_data == "video":
1658
+ return [gr.update(visible = False), gr.update(visible = False), gr.update(visible = False), gr.update(visible = True), gr.update(visible = False), gr.update(visible = True), gr.update(visible = True), gr.update(visible = True), gr.update(visible = True), gr.update(visible = True), gr.update(visible = True), gr.update(visible = False)]
1659
+
1660
+
1661
+ def handle_field_debug_change(input_image_debug_data, input_video_debug_data, prompt_debug_data, total_second_length_debug_data):
 
 
 
 
 
 
 
 
 
 
 
 
 
1662
  print("handle_field_debug_change")
1663
  input_image_debug_value[0] = input_image_debug_data
1664
  input_video_debug_value[0] = input_video_debug_data
 
1665
  prompt_debug_value[0] = prompt_debug_data
1666
  total_second_length_debug_value[0] = total_second_length_debug_data
1667
  return []
1668
 
1669
  input_image_debug.upload(
1670
  fn=handle_field_debug_change,
1671
+ inputs=[input_image_debug, input_video_debug, prompt_debug, total_second_length_debug],
1672
  outputs=[]
1673
  )
1674
 
1675
  input_video_debug.upload(
1676
  fn=handle_field_debug_change,
1677
+ inputs=[input_image_debug, input_video_debug, prompt_debug, total_second_length_debug],
 
 
 
 
 
 
1678
  outputs=[]
1679
  )
1680
 
1681
  prompt_debug.change(
1682
  fn=handle_field_debug_change,
1683
+ inputs=[input_image_debug, input_video_debug, prompt_debug, total_second_length_debug],
1684
  outputs=[]
1685
  )
1686
 
1687
  total_second_length_debug.change(
1688
  fn=handle_field_debug_change,
1689
+ inputs=[input_image_debug, input_video_debug, prompt_debug, total_second_length_debug],
1690
  outputs=[]
1691
  )
1692
 
 
1710
  generation_mode.change(
1711
  fn=handle_generation_mode_change,
1712
  inputs=[generation_mode],
1713
+ outputs=[text_to_video_hint, image_position, input_image, input_video, start_button, start_button_video, no_resize, batch, num_clean_frames, vae_batch, prompt_hint, fps_number]
1714
  )
1715
 
1716
  # Update display when the page loads
 
1718
  fn=handle_generation_mode_change, inputs = [
1719
  generation_mode
1720
  ], outputs = [
1721
+ text_to_video_hint, image_position, input_image, input_video, start_button, start_button_video, no_resize, batch, num_clean_frames, vae_batch, prompt_hint, fps_number
1722
  ]
1723
  )
1724