generate animation with frame images

app.py

@@ -11,6 +11,7 @@ with gr.Blocks() as demo:
         with gr.Column():
             char_imgs = gr.Gallery(type="pil", label="Images of the Character")
             mocap = gr.Video(label="Motion-Capture Video")
+            frame_imgs = gr.Gallery(type="pil", label="Reference Images of Each Frame")
             poses = gr.JSON(label="Pose Coordinates")
             tr_steps = gr.Number(label="Training steps", value=10)
             inf_steps = gr.Number(label="Inference steps", value=10)
@@ -40,7 +41,7 @@ with gr.Blocks() as demo:
     )
     
     inference_btn.click(
-        run_inference, inputs=[char_imgs, mocap, tr_steps, inf_steps, fps, modelId, img_width, img_height, remove_bg, resize_inputs], outputs=[animation, frames, frames_thumb, pose_coords, reference]
+        run_inference, inputs=[char_imgs, mocap, frame_imgs, tr_steps, inf_steps, fps, modelId, img_width, img_height, remove_bg, resize_inputs], outputs=[animation, frames, frames_thumb, pose_coords, reference]
     )
 
     generate_frame_btn.click(

main.py

@@ -285,14 +285,17 @@ def prepare_inputs_train(images, bg_remove, dwpose, rembg_session):
     return in_img, in_pose, train_imgs, train_poses
         
 
-def prepare_inputs_inference(in_img, in_vid, fps, dwpose, rembg_session, bg_remove, resize_inputs, is_app=False):
+def prepare_inputs_inference(in_img, in_vid, frames, fps, dwpose, rembg_session, bg_remove, resize_inputs, is_app=False):
     progress=gr.Progress(track_tqdm=True)
 
     print("prepare_inputs_inference")
     
     in_pose, _ = get_pose(in_img, dwpose, "in_pose.png")
     
-    frames = extract_frames(in_vid, fps)
+    print(in_vid)
+    print(frames)
+    if in_vid:
+        frames = extract_frames(in_vid, fps)
     print("remove background", bg_remove)
     if bg_remove:
         in_img = removebg(in_img, rembg_session)
@@ -354,7 +357,7 @@ def prepare_inputs(images, in_vid, fps, bg_remove, dwpose, rembg_session, resize
     
     in_img, in_pose, train_imgs, train_poses = prepare_inputs_train(images, bg_remove, dwpose, rembg_session)
     
-    in_img, target_poses_cropped, _, _, _ = prepare_inputs_inference(in_img, in_vid, fps, dwpose, rembg_session, bg_remove, resize_inputs, is_app)
+    in_img, target_poses_cropped, _, _, _ = prepare_inputs_inference(in_img, in_vid, [], fps, dwpose, rembg_session, bg_remove, resize_inputs, is_app)
     
     
     return in_img, in_pose, train_imgs, train_poses, target_poses_cropped
@@ -1123,7 +1126,7 @@ def run_train(images, train_steps=100, modelId="fine_tuned_pcdms", bg_remove=Tru
     train(modelId, in_img, in_pose, train_imgs, train_poses, train_steps, pcdms_model, noise_scheduler, image_encoder_p, image_encoder_g, vae, unet, finetune, is_app)
 
 
-def run_inference(images, video_path, train_steps=100, inference_steps=10, fps=12, modelId="fine_tuned_pcdms", img_width=1920, img_height=1080, bg_remove=True, resize_inputs=True):
+def run_inference(images, video_path, frames, train_steps=100, inference_steps=10, fps=12, modelId="fine_tuned_pcdms", img_width=1920, img_height=1080, bg_remove=True, resize_inputs=True):
     finetune=True
     is_app=True
     
@@ -1135,8 +1138,10 @@ def run_inference(images, video_path, train_steps=100, inference_steps=10, fps=1
     
     images = [img[0] for img in images]
     in_img = images[0]
+    if frames:
+        frames = [img[0] for img in frames]
 
-    in_img, target_poses, in_pose, target_poses_coords, orig_frames = prepare_inputs_inference(in_img, video_path, fps, dwpose, rembg_session, bg_remove, resize_inputs, is_app)
+    in_img, target_poses, in_pose, target_poses_coords, orig_frames = prepare_inputs_inference(in_img, video_path, frames, fps, dwpose, rembg_session, bg_remove, resize_inputs, is_app)
     
     results = inference(modelId, in_img, in_pose, target_poses, inference_steps, None, vae, unet, image_encoder_p, is_app)
     #urls = save_temp_imgs(results)