evaluation script

app.py

@@ -1,4 +1,5 @@
 from main import run_app, run_train, run_inference, run_generate_frame, run_interpolate_frames
+from evaluate import run_evaluate
 
 import spaces
 from PIL import Image
@@ -29,12 +30,14 @@ with gr.Blocks() as demo:
             generate_frame_btn = gr.Button(value="Generate Frame")
             submit_btn = gr.Button(value="Generate")
             interp_btn = gr.Button(value="Interpolate Frames")
+            eval_btn = gr.Button(value="Evaluate")
         with gr.Column():
             animation = gr.Video(label="Result")
             frames = gr.Gallery(type="pil", label="Frames", format="png")
             frames_thumb = gr.Gallery(type="pil", label="Thumbnails", format="png")
             pose_coords = gr.JSON(label="Pose Coordinates")
             reference = gr.Gallery(type="pil", label="Reference Images", format="png")
+            eval_scores = gr.JSON(label="Evaluation Scores")
 
     submit_btn.click(
         run_app, inputs=[char_imgs, mocap, tr_steps, inf_steps, fps, remove_bg, resize_inputs], outputs=[animation, frames]
@@ -56,6 +59,10 @@ with gr.Blocks() as demo:
         run_interpolate_frames, inputs=[interp_frame1, interp_frame2, times_to_interp], outputs=[frames, frames_thumb]
     )
 
+    eval_btn.click(
+        run_interpolate_frames, inputs=[], outputs=[eval_scores]
+    )
+    
 
 demo.launch(share=True)
 

evaluate.py

@@ -1,4 +1,4 @@
-from main import extract_frames, run
+from main import extract_frames, run_eval #run
 
 from PIL import Image
 import numpy as np
@@ -10,6 +10,7 @@ import lpips
 from pytorch_fid.fid_score import calculate_fid_given_paths
 import os
 import json
+from huggingface_hub import snapshot_download
 
 # Convert PIL to numpy
 def pil_to_np(img):
@@ -74,7 +75,8 @@ def get_score(item, image_paths, video_path, train_steps=100, inference_steps=10
     for i, frame in enumerate(gt_frames):
         frame.save("out/"+item+"/frame_"+str(i)+".png")
     
-    results = run(images, video_path, train_steps=100, inference_steps=10, fps=12, bg_remove=False, finetune=True)
+    #results = run(images, video_path, train_steps=100, inference_steps=10, fps=12, bg_remove=False, finetune=True)
+    results = run_eval(images, video_path, train_steps=100, inference_steps=10, fps=12, modelId="fine_tuned_pcdms", img_width=1920, img_height=1080, bg_remove=False, resize_inputs=False)
     
     for i, result in enumerate(results):
         result.save("out/"+item+"/result_"+str(i)+".png")
@@ -138,49 +140,75 @@ def get_score(item, image_paths, video_path, train_steps=100, inference_steps=10
     
     
     
-
-items = ['sidewalk', 'aaa', 'azri', 'dead', 'frankgirl', 'kobold', 'ramona', 'renee', 'walk', 'woody']
-for item in items: 
-    if item in metrics:
-        continue
-    get_score(item, ['test/'+item+'/1.jpg', 'test/'+item+'/2.jpg', 'test/'+item+'/3.jpg'], 'test/'+item+'/v.mp4')
+def get_files(directory_path):
+    """
+    Returns a list of all files in the specified directory.
+    """
+    files = []
+    for entry in os.listdir(directory_path):
+        full_path = os.path.join(directory_path, entry)
+        if os.path.isfile(full_path):
+            files.append(entry)
+    return files
 
 
+def run_evaluate():
+    snapshot_download(repo_id="acmyu/KeyframesAI-eval", local_dir="test")
 
-ssim = []
-psnr = []
-lpips = []
-fid = []
-ssim2 = []
-psnr2 = []
-lpips2 = []
-fid2 = []
-for item in metrics.keys():
-    ssim.append(metrics[item]['ft']['ssim']['avg'])
-    psnr.append(metrics[item]['ft']['psnr']['avg'])
-    lpips.append(metrics[item]['ft']['lpips']['avg'])
-    fid.append(metrics[item]['ft']['fid']['avg'])
+    items = os.listdir('test') 
+    items = ['test/woody'] #['sidewalk', 'aaa', 'azri', 'dead', 'frankgirl', 'kobold', 'ramona', 'renee', 'walk', 'woody']
     
-    ssim2.append(metrics[item]['base']['ssim']['avg'])
-    psnr2.append(metrics[item]['base']['psnr']['avg'])
-    lpips2.append(metrics[item]['base']['lpips']['avg'])
-    fid2.append(metrics[item]['base']['fid']['avg'])
+    for item in items: 
+        if item in metrics:
+            continue
 
-    print(item)
-    print("SSIM:", metrics[item]['ft']['ssim']['avg'], metrics[item]['base']['ssim']['avg'])
-    print("PSNR:", metrics[item]['ft']['psnr']['avg'], metrics[item]['base']['psnr']['avg'])
-    print("LPIPS:", metrics[item]['ft']['lpips']['avg'], metrics[item]['base']['lpips']['avg'])
-    print("FID:", metrics[item]['ft']['fid']['avg'], metrics[item]['base']['fid']['avg'])
-
-print('Results:')
-print("SSIM:", sum(ssim)/len(ssim))
-print("PSNR:", sum(psnr)/len(psnr))
-print("LPIPS:", sum(lpips)/len(lpips))
-print("FID:", sum(fid)/len(fid))
-print('baseline:')
-print("SSIM:", sum(ssim2)/len(ssim2))
-print("PSNR:", sum(psnr2)/len(psnr2))
-print("LPIPS:", sum(lpips2)/len(lpips2))
-print("FID:", sum(fid2)/len(fid2))
+        name = os.path.basename(os.path.dirname(item))
+        print(name)
+        
+        files = get_files(item)
+        videos = (x for x in files if x.endswith('.mp4'))
+        if len(videos) == 1:
+            get_score(name, list(filter(lambda x: not x.endswith('.mp4'), files)), videos[0])
+            #get_score(item, ['test/'+item+'/1.jpg', 'test/'+item+'/2.jpg', 'test/'+item+'/3.jpg'], 'test/'+item+'/v.mp4')
+        else:
+            print('Error: mp4 not found')
+
+
+
+    ssim = []
+    psnr = []
+    lpips = []
+    fid = []
+    ssim2 = []
+    psnr2 = []
+    lpips2 = []
+    fid2 = []
+    for item in metrics.keys():
+        ssim.append(metrics[item]['ft']['ssim']['avg'])
+        psnr.append(metrics[item]['ft']['psnr']['avg'])
+        lpips.append(metrics[item]['ft']['lpips']['avg'])
+        fid.append(metrics[item]['ft']['fid']['avg'])
+        
+        ssim2.append(metrics[item]['base']['ssim']['avg'])
+        psnr2.append(metrics[item]['base']['psnr']['avg'])
+        lpips2.append(metrics[item]['base']['lpips']['avg'])
+        fid2.append(metrics[item]['base']['fid']['avg'])
+
+        print(item)
+        print("SSIM:", metrics[item]['ft']['ssim']['avg'], metrics[item]['base']['ssim']['avg'])
+        print("PSNR:", metrics[item]['ft']['psnr']['avg'], metrics[item]['base']['psnr']['avg'])
+        print("LPIPS:", metrics[item]['ft']['lpips']['avg'], metrics[item]['base']['lpips']['avg'])
+        print("FID:", metrics[item]['ft']['fid']['avg'], metrics[item]['base']['fid']['avg'])
+
+    print('Results:')
+    print("SSIM:", sum(ssim)/len(ssim))
+    print("PSNR:", sum(psnr)/len(psnr))
+    print("LPIPS:", sum(lpips)/len(lpips))
+    print("FID:", sum(fid)/len(fid))
+    print('baseline:')
+    print("SSIM:", sum(ssim2)/len(ssim2))
+    print("PSNR:", sum(psnr2)/len(psnr2))
+    print("LPIPS:", sum(lpips2)/len(lpips2))
+    print("FID:", sum(fid2)/len(fid2))
 
 

libs/film/eval/interpolator.py

@@ -149,9 +149,7 @@ class Interpolator:
     self._align = align or None
     self._block_shape = block_shape or None
 
-  def __del__(self):
-    tf.keras.backend.clear_session()
-  
+
   def interpolate(self, x0: np.ndarray, x1: np.ndarray,
                   dt: np.ndarray) -> np.ndarray:
     """Generates an interpolated frame between given two batches of frames.

main.py

@@ -1163,7 +1163,7 @@ def run_inference_impl(images, video_path, frames, train_steps=100, inference_st
         frames = [img[0] for img in frames]
 
     in_img, target_poses, in_pose, target_poses_coords, orig_frames = prepare_inputs_inference(in_img, video_path, frames, fps, dwpose, rembg_session, bg_remove, resize_inputs, is_app)
-    target_poses[0].save('inf_pose.png')
+    #target_poses[0].save('inf_pose.png')
 
     results = inference(modelId, in_img, in_pose, target_poses, inference_steps, None, vae, unet, image_encoder_p, is_app)
     #urls = save_temp_imgs(results)
@@ -1207,7 +1207,7 @@ def generate_frame(images, target_poses, train_steps=100, inference_steps=10, mo
     target_poses = [Image.fromarray(draw_openpose(pose, height=img_height, width=img_width, include_hands=True, include_face=False)) for pose in target_poses]
 
     in_img, target_poses, in_pose, target_poses_coords, orig_frames = prepare_inputs_inference(in_img, None, [], 12, dwpose, rembg_session, bg_remove, resize_inputs, is_app, target_poses)
-    target_poses[0].save('gen_pose.png')
+    #target_poses[0].save('gen_pose.png')
 
     results = inference(modelId, in_img, in_pose, target_poses, inference_steps, None, vae, unet, image_encoder_p, is_app)
     #urls = save_temp_imgs(results)
@@ -1249,6 +1249,29 @@ def run_app(images, video_path, train_steps=100, inference_steps=10, fps=12, bg_
     
     return out_vid+'.webm', results
 
+
+def run_eval(images, video_path, train_steps=100, inference_steps=10, fps=12, modelId="fine_tuned_pcdms", img_width=1920, img_height=1080, bg_remove=False, resize_inputs=False):
+    finetune=True
+    is_app=False
+
+    dwpose, rembg_session, pcdms_model, noise_scheduler, image_encoder_p, image_encoder_g, vae, unet = load_models()
+
+    run_train(images, train_steps, modelId, bg_remove, resize_inputs)
+    
+    images = [img[0] for img in images]
+    in_img = images[0]
+
+    in_img, target_poses, in_pose, target_poses_coords, orig_frames = prepare_inputs_inference(in_img, video_path, [], fps, dwpose, rembg_session, bg_remove, resize_inputs, is_app)
+
+    _, results, _, _, _ = run_inference_impl(images, video_path, frames, train_steps, inference_steps, fps, modelId, img_width, img_height, bg_remove, resize_inputs)
+
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    return results
+
+
+
 @spaces.GPU(duration=30)
 def interpolate_frames(frame1, frame2, times_to_interp):
     film = Predictor()