import os import cv2 import torch import importlib.util import sys import argparse import numpy as np from torch.nn import functional as F import warnings import _thread from queue import Queue, Empty from model.pytorch_msssim import ssim_matlab import time warnings.filterwarnings("ignore") loglevel = os.environ.get("RIFE_LOGLEVEL", "error") os.environ["SDL_AUDIODRIVER"] = "dummy" os.environ["ALSA_CONFIG_PATH"] = "/dev/null" # =============== CUSTOM PROGRESS TRACKER =============== def format_time(seconds: float) -> str: """Format seconds to H:MM:SS or MM:SS.""" m, s = divmod(int(seconds), 60) h, m = divmod(m, 60) if h > 0: return f"{h}:{m:02}:{s:02}" else: return f"{m:02}:{s:02}" class VideoProgressTracker: def __init__(self, total_frames): self.total_frames = total_frames self.current_frame = 0 self.start_time = time.time() def update(self, frame_num=None): if frame_num is not None: self.current_frame = frame_num else: self.current_frame += 1 self.display_progress() def display_progress(self): elapsed = time.time() - self.start_time progress_fraction = self.current_frame / self.total_frames if self.total_frames > 0 else 0 fps = self.current_frame / elapsed if elapsed > 0 else 0 eta = (elapsed / progress_fraction - elapsed) if progress_fraction > 0 else 0 percent = int(progress_fraction * 100) info = (f"Interpolating: {percent:3}% " f"Frame {self.current_frame}/{self.total_frames} | " f"Elapsed: {format_time(elapsed)} | ETA: {format_time(eta)} | {fps:.1f} frame/s") sys.stdout.write('\r' + info) sys.stdout.flush() def finish(self): """Finalize progress display""" # Show 100% completion before finishing self.current_frame = self.total_frames self.display_progress() sys.stdout.write('\n') sys.stdout.flush() def transferAudio(sourceVideo, targetVideo): import shutil import moviepy.editor tempAudioFileName = "./temp/audio.mkv" if True: if os.path.isdir("temp"): shutil.rmtree("temp") os.makedirs("temp") os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -c:a copy -vn {}'.format(loglevel, sourceVideo, tempAudioFileName)) targetNoAudio = os.path.splitext(targetVideo)[0] + "_noaudio" + os.path.splitext(targetVideo)[1] os.rename(targetVideo, targetNoAudio) os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -i {} -c copy "{}"'.format(loglevel, targetNoAudio, tempAudioFileName, targetVideo)) if os.path.getsize(targetVideo) == 0: tempAudioFileName = "./temp/audio.m4a" os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -c:a aac -b:a 160k -vn {}'.format(loglevel, sourceVideo, tempAudioFileName)) os.system('ffmpeg -hide_banner -loglevel {} -y -i "{}" -i {} -c copy "{}"'.format(loglevel, targetNoAudio, tempAudioFileName, targetVideo)) if (os.path.getsize(targetVideo) == 0): os.rename(targetNoAudio, targetVideo) print("Audio transfer failed. Interpolated video will have no audio") else: print("Lossless audio transfer failed. Audio was transcoded to AAC (M4A) instead.") os.remove(targetNoAudio) else: os.remove(targetNoAudio) shutil.rmtree("temp") parser = argparse.ArgumentParser(description='Interpolation for a pair of images') parser.add_argument('--video', dest='video', type=str, default=None) parser.add_argument('--output', dest='output', type=str, default=None) parser.add_argument('--img', dest='img', type=str, default=None) parser.add_argument('--montage', dest='montage', action='store_true', help='montage origin video') parser.add_argument('--model', dest='modelDir', type=str, default='train_log', help='directory with trained model files') parser.add_argument('--interpolation_factor', type=int, default=2, help="How many total frames between two input frames") parser.add_argument('--mode', type=str, choices=['fast', 'slow'], default='slow', help="Interpolation mode: 'fast uses multi' (simple split) or 'slow uses exp' (recursive)") parser.add_argument('--UHD', dest='UHD', action='store_true', help='support 4k video') parser.add_argument('--scale', dest='scale', type=float, default=1.0, help='Try scale=0.5 for 4k video') parser.add_argument('--skip', dest='skip', action='store_true', help='whether to remove static frames before processing') parser.add_argument('--fps', dest='fps', type=int, default=None) parser.add_argument('--png', dest='png', action='store_true', help='whether to vid_out png format vid_outs') parser.add_argument('--ext', dest='ext', type=str, default='mp4', help='vid_out video extension') args = parser.parse_args() args.multi = args.interpolation_factor assert (not args.video is None or not args.img is None) if args.skip: print("skip flag is abandoned, please refer to issue #207.") if args.UHD and args.scale==1.0: args.scale = 0.5 assert args.scale in [0.25, 0.5, 1.0, 2.0, 4.0] if not args.img is None: args.png = True device = torch.device("cuda" if torch.cuda.is_available() else "cpu") torch.set_grad_enabled(False) if torch.cuda.is_available(): torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = True # Set Path to Practical-RIFE args.modelDir = os.path.join("/content/Practical-RIFE", args.modelDir) args.modelDir = os.path.abspath(args.modelDir) sys.path.insert(0, args.modelDir) model_path = os.path.join(args.modelDir, 'RIFE_HDv3.py') spec = importlib.util.spec_from_file_location("RIFE_HDv3", model_path) RIFE_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(RIFE_module) Model = RIFE_module.Model model = Model() if not hasattr(model, 'version'): model.version = 0 model.load_model(args.modelDir, -1) model.eval() model.device() model_name = os.path.basename(os.path.abspath(args.modelDir)) if not args.video is None: videoCapture = cv2.VideoCapture(args.video) fps = videoCapture.get(cv2.CAP_PROP_FPS) tot_frame = videoCapture.get(cv2.CAP_PROP_FRAME_COUNT) videoCapture.release() if args.fps is None: fpsNotAssigned = True args.fps = fps * args.interpolation_factor else: fpsNotAssigned = False videoCapture = cv2.VideoCapture(args.video) success, lastframe = videoCapture.read() if success: lastframe = cv2.cvtColor(lastframe, cv2.COLOR_BGR2RGB) videogen = [] while success: videogen.append(lastframe) success, lastframe = videoCapture.read() if success: lastframe = cv2.cvtColor(lastframe,cv2.COLOR_BGR2RGB) videoCapture.release() lastframe = videogen.pop(0) fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v') video_path_wo_ext, ext = os.path.splitext(args.video) file_name = os.path.basename(args.video) print(f"{tot_frame} frames in total, {fps}FPS to {fps * args.interpolation_factor}FPS\n") if args.png == False and fpsNotAssigned == True: pass else: pass else: videogen = [] for f in os.listdir(args.img): if 'png' in f: videogen.append(f) tot_frame = len(videogen) videogen.sort(key= lambda x:int(x[:-4])) lastframe = cv2.imread(os.path.join(args.img, videogen[0]), cv2.IMREAD_UNCHANGED)[:, :, ::-1].copy() videogen = videogen[1:] folder_path = os.path.abspath(args.img) print(f"{len(videogen)} PNG frames found.\n") h, w, _ = lastframe.shape vid_out_name = None vid_out = None if args.png: if not os.path.exists('vid_out'): os.mkdir('vid_out') else: if args.output is not None: vid_out_name = args.output else: vid_out_name = '{}_{}X_{}fps.{}'.format(video_path_wo_ext, args.interpolation_factor, int(np.round(args.fps)), args.ext) vid_out = cv2.VideoWriter(vid_out_name, fourcc, args.fps, (w, h)) def clear_write_buffer(user_args, write_buffer): cnt = 0 while True: item = write_buffer.get() if item is None: break if user_args.png: cv2.imwrite('vid_out/{:0>7d}.png'.format(cnt), item[:, :, ::-1]) cnt += 1 else: vid_out.write(item[:, :, ::-1]) def build_read_buffer(user_args, read_buffer, videogen): try: for frame in videogen: if not user_args.img is None: frame = cv2.imread(os.path.join(user_args.img, frame), cv2.IMREAD_UNCHANGED)[:, :, ::-1].copy() if user_args.montage: frame = frame[:, left: left + w] read_buffer.put(frame) except: pass read_buffer.put(None) #OriginalLogic def make_inference(I0, I1, n): global model if args.mode == "slow": if n == 1: middle = model.inference(I0, I1, 0.5, args.scale) return [middle] middle = model.inference(I0, I1, 0.5, args.scale) left_half = make_inference(I0, middle, n // 2) right_half = make_inference(middle, I1, n // 2) if n % 2: return [*left_half, middle, *right_half] else: return [*left_half, *right_half] else: outputs = [] for i in range(n): timestep = (i + 1) / (n + 1) middle = model.inference(I0, I1, timestep, args.scale) outputs.append(middle) return outputs def pad_image(img, padding): if any(padding): img = F.pad(img, padding) return img if args.montage: left = w // 4 w = w // 2 tmp = max(128, int(128 / args.scale)) ph = ((h - 1) // tmp + 1) * tmp pw = ((w - 1) // tmp + 1) * tmp padding = (0, pw - w, 0, ph - h) # Initialize custom progress tracker progress = VideoProgressTracker(int(tot_frame)) if args.montage: lastframe = lastframe[:, left: left + w] write_buffer = Queue(maxsize=500) read_buffer = Queue(maxsize=500) _thread.start_new_thread(build_read_buffer, (args, read_buffer, videogen)) _thread.start_new_thread(clear_write_buffer, (args, write_buffer)) I1 = torch.from_numpy(np.transpose(lastframe, (2,0,1))).to(device, non_blocking=True).unsqueeze(0).float() / 255. I1 = pad_image(I1, padding) temp = None while True: if temp is not None: frame = temp temp = None else: frame = read_buffer.get() if frame is None: break I0 = I1 I1 = torch.from_numpy(np.transpose(frame, (2,0,1))).to(device, non_blocking=True).unsqueeze(0).float() / 255. I1 = pad_image(I1, padding) I0_small = F.interpolate(I0, (32, 32), mode='bilinear', align_corners=False) I1_small = F.interpolate(I1, (32, 32), mode='bilinear', align_corners=False) ssim = ssim_matlab(I0_small[:, :3], I1_small[:, :3]) break_flag = False if ssim > 0.996: frame = read_buffer.get() if frame is None: break_flag = True frame = lastframe else: temp = frame I1 = torch.from_numpy(np.transpose(frame, (2,0,1))).to(device, non_blocking=True).unsqueeze(0).float() / 255. I1 = pad_image(I1, padding) I1 = model.inference(I0, I1, scale=args.scale) I1_small = F.interpolate(I1, (32, 32), mode='bilinear', align_corners=False) ssim = ssim_matlab(I0_small[:, :3], I1_small[:, :3]) frame = (I1[0] * 255).byte().cpu().numpy().transpose(1, 2, 0)[:h, :w] if ssim < 0.2: output = [] for i in range(args.interpolation_factor - 1): output.append(I0) else: output = make_inference(I0, I1, args.interpolation_factor - 1) if args.montage: write_buffer.put(np.concatenate((lastframe, lastframe), 1)) for mid in output: mid = (((mid[0] * 255.).byte().cpu().numpy().transpose(1, 2, 0))) write_buffer.put(np.concatenate((lastframe, mid[:h, :w]), 1)) else: write_buffer.put(lastframe) for mid in output: mid = (((mid[0] * 255.).byte().cpu().numpy().transpose(1, 2, 0))) write_buffer.put(mid[:h, :w]) progress.update() lastframe = frame if break_flag: break if args.montage: write_buffer.put(np.concatenate((lastframe, lastframe), 1)) else: write_buffer.put(lastframe) progress.update() # Add progress update for final frame write_buffer.put(None) while(not write_buffer.empty()): time.sleep(0.1) progress.finish() if not vid_out is None: vid_out.release() if args.png == False and fpsNotAssigned == True and not args.video is None: try: transferAudio(args.video, vid_out_name) except: print("Audio transfer failed. Interpolated video will have no audio") targetNoAudio = os.path.splitext(vid_out_name)[0] + "_noaudio" + os.path.splitext(vid_out_name)[1] os.rename(targetNoAudio, vid_out_name)