Upload 23 files

.gitattributes

@@ -483,3 +483,4 @@ custom_nodes/ComfyUI-SeedVR2_VideoUpscaler/docs/usage_01.png filter=lfs diff=lfs
 custom_nodes/ComfyUI-SeedVR2_VideoUpscaler/docs/usage_02.png filter=lfs diff=lfs merge=lfs -text
 custom_nodes/ComfyUI-SeedVR2_VideoUpscaler/example_workflows/example_inputs/Mustache_640x360.mp4 filter=lfs diff=lfs merge=lfs -text
 custom_nodes/ComfyUI-SeedVR2_VideoUpscaler/example_workflows/example_inputs/Sadhu_320x478.png filter=lfs diff=lfs merge=lfs -text
+custom_nodes/ComfyUI-VFI/docs/image.png filter=lfs diff=lfs merge=lfs -text

custom_nodes/ComfyUI-VFI/.gitignore

@@ -0,0 +1,7 @@
+*.pkl
+*.safetensors
+**/*.pkl
+**/*.safetensors
+**/__pycache__
+.dev
+.DS_Store

custom_nodes/ComfyUI-VFI/README.md

@@ -0,0 +1,62 @@
+# ComfyUI-VFI
+
+Video Frame Interpolation nodes for ComfyUI using RIFE (Real-Time Intermediate Flow Estimation).
+
+![RIFE](./docs/image.png)
+
+## Features
+
+- High-quality frame interpolation using RIFE
+- Convert between different frame rates (e.g., 30fps to 60fps)
+- Adjustable processing scale for performance/quality trade-off
+- Model caching for efficient processing
+- Progress tracking in ComfyUI
+
+## Installation
+
+- Clone this repository into your ComfyUI custom_nodes directory:
+
+```bash
+cd ComfyUI/custom_nodes
+git clone https://github.com/your-username/ComfyUI-VFI.git
+```
+
+- Install required dependencies:
+
+```bash
+cd ComfyUI-VFI
+pip install -r requirements.txt
+```
+
+- The RIFE model will be automatically downloaded on first use
+  - Alternatively, you can manually place `flownet.pkl` in:
+    - `ComfyUI-VFI/rife/train_log/`
+    - Or `ComfyUI/models/rife/`
+
+## Usage
+
+The node will appear in the "image/animation" category as "RIFE Frame Interpolation".
+
+### Inputs
+
+- **images**: Image sequence tensor [N, H, W, C]
+- **source_fps**: Original frame rate (default: 30.0)
+- **target_fps**: Desired frame rate (default: 60.0)
+- **scale**: Processing scale factor (default: 1.0)
+  - Lower values (0.25-0.5) for faster processing
+  - Higher values (1.0-4.0) for better quality
+
+### Output
+
+- **images**: Interpolated image sequence tensor
+
+## Example Workflow
+
+1. Load video frames using a video loader node
+2. Connect to RIFE Frame Interpolation node
+3. Set source and target FPS
+4. Connect output to video encoder or preview
+
+## Model Download
+
+The RIFE model (`flownet.pkl`) can be downloaded from the official RIFE repository.

custom_nodes/ComfyUI-VFI/__init__.py

@@ -0,0 +1,5 @@
+"""ComfyUI-VFI: Video Frame Interpolation nodes for ComfyUI"""
+
+from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
+
+__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]

custom_nodes/ComfyUI-VFI/examples/interp.json

@@ -0,0 +1,343 @@
+{
+  "id": "68247ae1-7321-451f-9b71-b44476309c39",
+  "revision": 0,
+  "last_node_id": 19,
+  "last_link_id": 19,
+  "nodes": [
+    {
+      "id": 13,
+      "type": "VHS_VideoInfoLoaded",
+      "pos": [585.9952392578125, 234.33438110351562],
+      "size": [247.837890625, 106],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "video_info",
+          "type": "VHS_VIDEOINFO",
+          "link": 12
+        }
+      ],
+      "outputs": [
+        {
+          "name": "fps🟦",
+          "type": "FLOAT",
+          "links": [13, 15]
+        },
+        {
+          "name": "frame_count🟦",
+          "type": "INT",
+          "links": null
+        },
+        {
+          "name": "duration🟦",
+          "type": "FLOAT",
+          "links": null
+        },
+        {
+          "name": "width🟦",
+          "type": "INT",
+          "links": null
+        },
+        {
+          "name": "height🟦",
+          "type": "INT",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VHS_VideoInfoLoaded"
+      },
+      "widgets_values": {}
+    },
+    {
+      "id": 15,
+      "type": "easy showAnything",
+      "pos": [688.3895874023438, 443.7354736328125],
+      "size": [210, 88],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "anything",
+          "shape": 7,
+          "type": "*",
+          "link": 15
+        }
+      ],
+      "outputs": [
+        {
+          "name": "output",
+          "type": "*",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "easy showAnything"
+      },
+      "widgets_values": ["32.0"]
+    },
+    {
+      "id": 16,
+      "type": "VHS_VideoCombine",
+      "pos": [1325.4146728515625, 65.38612365722656],
+      "size": [220.5830078125, 436.82794189453125],
+      "flags": {},
+      "order": 6,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 16
+        },
+        {
+          "name": "audio",
+          "shape": 7,
+          "type": "AUDIO",
+          "link": null
+        },
+        {
+          "name": "meta_batch",
+          "shape": 7,
+          "type": "VHS_BatchManager",
+          "link": null
+        },
+        {
+          "name": "vae",
+          "shape": 7,
+          "type": "VAE",
+          "link": null
+        },
+        {
+          "name": "frame_rate",
+          "type": "FLOAT",
+          "widget": {
+            "name": "frame_rate"
+          },
+          "link": 18
+        }
+      ],
+      "outputs": [
+        {
+          "name": "Filenames",
+          "type": "VHS_FILENAMES",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VHS_VideoCombine"
+      },
+      "widgets_values": {
+        "frame_rate": 8,
+        "loop_count": 0,
+        "filename_prefix": "AnimateDiff",
+        "format": "video/h264-mp4",
+        "pix_fmt": "yuv420p",
+        "crf": 19,
+        "save_metadata": true,
+        "pingpong": false,
+        "save_output": true,
+        "videopreview": {
+          "hidden": false,
+          "paused": false,
+          "params": {
+            "filename": "AnimateDiff_00594.mp4",
+            "subfolder": "",
+            "type": "output",
+            "format": "video/h264-mp4",
+            "frame_rate": 24
+          },
+          "muted": false
+        }
+      }
+    },
+    {
+      "id": 18,
+      "type": "PrimitiveFloat",
+      "pos": [930.571533203125, 273.9353942871094],
+      "size": [270, 58],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {
+          "name": "FLOAT",
+          "type": "FLOAT",
+          "links": [17, 18, 19]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "PrimitiveFloat"
+      },
+      "widgets_values": [24]
+    },
+    {
+      "id": 10,
+      "type": "VHS_LoadVideo",
+      "pos": [278.06610107421875, 158.2479248046875],
+      "size": [253.279296875, 262],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "meta_batch",
+          "shape": 7,
+          "type": "VHS_BatchManager",
+          "link": null
+        },
+        {
+          "name": "vae",
+          "shape": 7,
+          "type": "VAE",
+          "link": null
+        }
+      ],
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [14]
+        },
+        {
+          "name": "frame_count",
+          "type": "INT",
+          "links": null
+        },
+        {
+          "name": "audio",
+          "type": "AUDIO",
+          "links": null
+        },
+        {
+          "name": "video_info",
+          "type": "VHS_VIDEOINFO",
+          "links": [12]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "VHS_LoadVideo"
+      },
+      "widgets_values": {
+        "video": "TWYI-PKSR-A0TV-H5VK-5H7U_2X_32fps.mp4",
+        "force_rate": 0,
+        "force_size": "Disabled",
+        "custom_width": 512,
+        "custom_height": 512,
+        "frame_load_cap": 0,
+        "skip_first_frames": 0,
+        "select_every_nth": 1,
+        "choose video to upload": "image",
+        "videopreview": {
+          "hidden": false,
+          "paused": false,
+          "params": {
+            "frame_load_cap": 0,
+            "skip_first_frames": 0,
+            "force_rate": 0,
+            "select_every_nth": 1,
+            "filename": "TWYI-PKSR-A0TV-H5VK-5H7U_2X_32fps.mp4",
+            "type": "input",
+            "format": "video/mp4"
+          },
+          "muted": false
+        }
+      }
+    },
+    {
+      "id": 14,
+      "type": "RIFEInterpolation",
+      "pos": [943.4434204101562, 62.89570617675781],
+      "size": [270, 130],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "link": 14
+        },
+        {
+          "name": "source_fps",
+          "type": "FLOAT",
+          "widget": {
+            "name": "source_fps"
+          },
+          "link": 13
+        },
+        {
+          "name": "target_fps",
+          "type": "FLOAT",
+          "widget": {
+            "name": "target_fps"
+          },
+          "link": 17
+        }
+      ],
+      "outputs": [
+        {
+          "name": "images",
+          "type": "IMAGE",
+          "links": [16]
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "RIFEInterpolation"
+      },
+      "widgets_values": [30, 24, 1, "flownet.pkl"]
+    },
+    {
+      "id": 19,
+      "type": "easy showAnything",
+      "pos": [1063.455078125, 438.5444641113281],
+      "size": [140, 76],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "anything",
+          "shape": 7,
+          "type": "*",
+          "link": 19
+        }
+      ],
+      "outputs": [
+        {
+          "name": "output",
+          "type": "*",
+          "links": null
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "easy showAnything"
+      },
+      "widgets_values": ["24.0"]
+    }
+  ],
+  "links": [
+    [12, 10, 3, 13, 0, "VHS_VIDEOINFO"],
+    [13, 13, 0, 14, 1, "FLOAT"],
+    [14, 10, 0, 14, 0, "IMAGE"],
+    [15, 13, 0, 15, 0, "*"],
+    [16, 14, 0, 16, 0, "IMAGE"],
+    [17, 18, 0, 14, 2, "FLOAT"],
+    [18, 18, 0, 16, 4, "FLOAT"],
+    [19, 18, 0, 19, 0, "*"]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 1.1000000000000005,
+      "offset": [-218.49763739436602, 25.55563144593943]
+    },
+    "frontendVersion": "1.19.9"
+  },
+  "version": 0.4
+}

custom_nodes/ComfyUI-VFI/nodes.py

@@ -0,0 +1,297 @@
+"""ComfyUI nodes for Video Frame Interpolation using RIFE"""
+
+import os
+import subprocess
+import sys
+
+import torch
+
+from .rife.rife_comfyui_wrapper import RIFEWrapper
+
+try:
+    import comfy.utils
+    import folder_paths
+except ImportError:
+    folder_paths = None
+    comfy = None
+
+
+MODEL_CACHE = {}
+
+
+class RIFEInterpolation:
+    """
+    ComfyUI node for RIFE (Real-Time Intermediate Flow Estimation) video frame interpolation.
+    Takes a sequence of images and interpolates frames to achieve a target frame rate.
+    """
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "images": ("IMAGE",),
+                "source_fps": (
+                    "FLOAT",
+                    {
+                        "default": 30.0,
+                        "min": 1.0,
+                        "max": 120.0,
+                        "step": 0.1,
+                        "display": "number",
+                        "tooltip": "Source video frame rate",
+                    },
+                ),
+                "target_fps": (
+                    "FLOAT",
+                    {
+                        "default": 60.0,
+                        "min": 1.0,
+                        "max": 240.0,
+                        "step": 0.1,
+                        "display": "number",
+                        "tooltip": "Target frame rate after interpolation",
+                    },
+                ),
+                "scale": (
+                    "FLOAT",
+                    {
+                        "default": 1.0,
+                        "min": 0.25,
+                        "max": 4.0,
+                        "step": 0.25,
+                        "display": "number",
+                        "tooltip": "Processing scale factor. Lower values process faster but may reduce quality",
+                    },
+                ),
+            },
+            "optional": {
+                "model_name": (
+                    ["flownet.pkl"],
+                    {"default": "flownet.pkl", "tooltip": "RIFE model to use for interpolation"},
+                ),
+                "batch_size": (
+                    "INT",
+                    {
+                        "default": 8,
+                        "min": 1,
+                        "max": 32,
+                        "step": 1,
+                        "display": "number",
+                        "tooltip": "Number of frames to process in parallel. Higher values are faster but use more VRAM",
+                    },
+                ),
+                "use_fp16": (
+                    "BOOLEAN",
+                    {
+                        "default": True,
+                        "tooltip": "Use half precision (FP16) for faster inference and lower VRAM usage. Requires CUDA GPU",
+                    },
+                ),
+            },
+        }
+
+    RETURN_TYPES = ("IMAGE",)
+    RETURN_NAMES = ("images",)
+
+    FUNCTION = "interpolate"
+
+    CATEGORY = "image/animation"
+
+    DESCRIPTION = "Interpolate video frames using RIFE (Real-Time Intermediate Flow Estimation) to increase frame rate"
+
+    def interpolate(self, images, source_fps, target_fps, scale, model_name="flownet.pkl", batch_size=8, use_fp16=True):
+        # Validate inputs
+        if images is None or len(images) == 0:
+            raise ValueError("No images provided")
+
+        if len(images.shape) != 4 or images.shape[-1] != 3:
+            raise ValueError(f"Expected image tensor shape [N, H, W, 3], got {images.shape}")
+
+        if source_fps <= 0 or target_fps <= 0:
+            raise ValueError("Frame rates must be positive")
+
+        if scale <= 0:
+            raise ValueError("Scale must be positive")
+
+        # If source and target fps are the same, return original
+        if abs(source_fps - target_fps) < 0.01:
+            return (images,)
+
+        # Get or load model
+        model = self._get_or_load_model(model_name, use_fp16=use_fp16)
+
+        duration = len(images) / source_fps
+        total_target_frames = int(duration * target_fps)
+
+        pbar = None
+        if comfy and hasattr(comfy, "utils"):
+            pbar = comfy.utils.ProgressBar(total_target_frames)
+
+        def progress_callback(current, total):
+            if pbar:
+                pbar.update_absolute(current, total)
+
+        # Use autocast context for mixed precision
+        autocast_enabled = use_fp16 and torch.cuda.is_available()
+        autocast_context = torch.amp.autocast("cuda") if autocast_enabled else torch.nullcontext()
+
+        try:
+            with autocast_context:
+                interpolated_images = model.interpolate_frames(
+                    images=images,
+                    source_fps=source_fps,
+                    target_fps=target_fps,
+                    scale=scale,
+                    progress_callback=progress_callback,
+                    batch_size=batch_size,
+                )
+
+            return (interpolated_images,)
+
+        except Exception as e:
+            raise RuntimeError(f"Frame interpolation failed: {str(e)}")
+
+    def _get_or_load_model(self, model_name, use_fp16=False):
+        """Load model from cache or disk"""
+        global MODEL_CACHE
+
+        # Create cache key with fp16 flag
+        cache_key = f"{model_name}_fp16" if use_fp16 else model_name
+
+        if cache_key in MODEL_CACHE:
+            return MODEL_CACHE[cache_key]
+
+        # Look for model in multiple locations
+        model_paths = [
+            os.path.join(os.path.dirname(__file__), "rife", "train_log", model_name),
+            os.path.join(os.path.dirname(__file__), "models", model_name),
+        ]
+
+        # Add ComfyUI model directory if available
+        if folder_paths and hasattr(folder_paths, "models_dir"):
+            model_paths.insert(1, os.path.join(folder_paths.models_dir, "rife", model_name))
+
+        model_path = None
+        for path in model_paths:
+            if os.path.exists(path):
+                model_path = path
+                break
+
+        if model_path is None:
+            # Try to download the model automatically
+            print(f"RIFE model '{model_name}' not found. Attempting to download...")
+
+            # Default download location
+            download_target = os.path.join(os.path.dirname(__file__), "rife", "train_log")
+
+            try:
+                # Run the download script
+                download_script = os.path.join(os.path.dirname(__file__), "rife", "download_rife.py")
+
+                if os.path.exists(download_script):
+                    result = subprocess.run(
+                        [sys.executable, download_script, download_target], capture_output=True, text=True
+                    )
+
+                    if result.returncode == 0:
+                        print("Model downloaded successfully!")
+                        # Check if model now exists
+                        model_path = os.path.join(download_target, model_name)
+                        if not os.path.exists(model_path):
+                            raise FileNotFoundError(
+                                f"Model download completed but '{model_name}' not found at expected location."
+                            )
+                    else:
+                        raise RuntimeError(f"Model download failed: {result.stderr}")
+                else:
+                    raise FileNotFoundError(
+                        f"Download script not found at {download_script}. "
+                        f"Please manually download the model and place it in one of these locations:\n"
+                        + "\n".join(f"  - {p}" for p in model_paths)
+                    )
+
+            except Exception as e:
+                raise RuntimeError(
+                    f"Failed to automatically download RIFE model: {str(e)}\n"
+                    f"Please manually download the model and place it in one of these locations:\n"
+                    + "\n".join(f"  - {p}" for p in model_paths)
+                )
+
+        # Load model
+        print(f"Loading RIFE model from: {model_path}")
+        model = RIFEWrapper(model_path, use_fp16=use_fp16)
+        MODEL_CACHE[cache_key] = model
+
+        return model
+
+    @classmethod
+    def IS_CHANGED(cls, **kwargs):
+        return float("NaN")
+
+
+class CalculateLoadedFPS:
+    """
+    计算加载后的FPS，根据原始FPS和每n帧选择一帧的参数
+    Calculate loaded FPS based on source FPS and select_every_nth parameter
+    """
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "source_fps": (
+                    "FLOAT",
+                    {
+                        "default": 24,
+                        "min": 0.1,
+                        "max": 160.0,
+                        "step": 0.1,
+                        "display": "number",
+                        "tooltip": "Source video frame rate",
+                    },
+                ),
+                "select_every_nth": (
+                    "INT",
+                    {
+                        "default": 1,
+                        "min": 1,
+                        "max": 100,
+                        "step": 1,
+                        "display": "number",
+                        "tooltip": "Select every Nth frame (from VideoHelperSuite)",
+                    },
+                ),
+            },
+        }
+
+    RETURN_TYPES = ("FLOAT",)
+    RETURN_NAMES = ("loaded_fps",)
+
+    FUNCTION = "calculate_fps"
+
+    CATEGORY = "image/animation"
+
+    DESCRIPTION = "Calculate loaded FPS after frame selection (source_fps / select_every_nth)"
+
+    def calculate_fps(self, source_fps, select_every_nth):
+        # 验证输入
+        if source_fps <= 0:
+            raise ValueError("source_fps must be positive")
+        if select_every_nth <= 0:
+            raise ValueError("select_every_nth must be positive")
+
+        # 计算加载后的FPS
+        loaded_fps = source_fps / select_every_nth
+        return (loaded_fps,)
+
+
+# ComfyUI node mappings
+NODE_CLASS_MAPPINGS = {
+    "RIFEInterpolation": RIFEInterpolation,
+    "CalculateLoadedFPS": CalculateLoadedFPS,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "RIFEInterpolation": "RIFE Frame Interpolation",
+    "CalculateLoadedFPS": "Calculate Loaded FPS",
+}

custom_nodes/ComfyUI-VFI/pyproject.toml

@@ -0,0 +1,25 @@
+[project]
+name = "rife_comfyui_wrapper"
+description = "ComfyUI-RIFE is an inference wrapper for RIFE designed for use with ComfyUI."
+version = "1.0.0"
+license = { file = "LICENSE" }
+
+[project.urls]
+Repository = "https://github.com/ModelTC/ComfyUI-VFI"
+#  Used by Comfy Registry https://comfyregistry.org
+
+[tool.comfy]
+PublisherId = "gaclove"
+DisplayName = "ComfyUI-VFI"
+Icon = ""
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+extend-select = ["I"]
+
+[tool.ruff.lint.per-file-ignores]
+
+"rife/train_log/RIFE_HDv3.py" = ["F"]
+"rife/train_log/IFNet_HDv3.py" = ["F"]

custom_nodes/ComfyUI-VFI/requirements.txt

@@ -0,0 +1,4 @@
+torch>=2.0.0
+torchvision>=0.15.0
+numpy>=1.21.0
+requests>=2.25.0

custom_nodes/ComfyUI-VFI/rife/download_rife.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+import argparse
+import os
+import shutil
+import sys
+import zipfile
+from pathlib import Path
+
+import requests
+
+
+def is_huggingface_accessible():
+    try:
+        _response = requests.get("https://huggingface.co", timeout=3)
+        return True
+    except (requests.ConnectionError, requests.Timeout):
+        return False
+
+
+def get_base_dir():
+    """Get project root directory"""
+    return Path(__file__).parent.parent
+
+
+def download_file(url, save_path):
+    """Download file"""
+    print(f"Starting download: {url}")
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+
+    total_size = int(response.headers.get("content-length", 0))
+    downloaded_size = 0
+
+    with open(save_path, "wb") as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                f.write(chunk)
+                downloaded_size += len(chunk)
+                if total_size > 0:
+                    progress = (downloaded_size / total_size) * 100
+                    print(f"\rDownload progress: {progress:.1f}%", end="", flush=True)
+
+    print(f"\nDownload completed: {save_path}")
+
+
+def extract_zip(zip_path, extract_to):
+    """Extract zip file"""
+    print(f"Starting extraction: {zip_path}")
+    with zipfile.ZipFile(zip_path, "r") as zip_ref:
+        zip_ref.extractall(extract_to)
+    print(f"Extraction completed: {extract_to}")
+
+
+def find_flownet_pkl(extract_dir):
+    """Find flownet.pkl file in extracted directory"""
+    for root, _dirs, files in os.walk(extract_dir):
+        for file in files:
+            if file == "flownet.pkl":
+                return os.path.join(root, file)
+    return None
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Download RIFE model to specified directory")
+    parser.add_argument("target_directory", help="Target directory path")
+
+    args = parser.parse_args()
+
+    target_dir = Path(args.target_directory)
+    if not target_dir.is_absolute():
+        target_dir = Path.cwd() / target_dir
+
+    base_dir = get_base_dir()
+    temp_dir = base_dir / "_temp"
+
+    # Create temporary directory
+    temp_dir.mkdir(exist_ok=True)
+
+    target_dir.mkdir(parents=True, exist_ok=True)
+
+    if not is_huggingface_accessible():
+        print("huggingface.co is not accessible, using hf-mirror.com")
+        zip_url = "https://hf-mirror.com/hzwer/RIFE/resolve/main/RIFEv4.26_0921.zip"
+    else:
+        zip_url = "https://huggingface.co/hzwer/RIFE/resolve/main/RIFEv4.26_0921.zip"
+
+    zip_path = temp_dir / "RIFEv4.26_0921.zip"
+
+    try:
+        download_file(zip_url, zip_path)
+        extract_zip(zip_path, temp_dir)
+        flownet_pkl = find_flownet_pkl(temp_dir)
+        if flownet_pkl:
+            target_file = target_dir / "flownet.pkl"
+            shutil.copy2(flownet_pkl, target_file)
+            print(f"flownet.pkl copied to: {target_file}")
+        else:
+            print("Error: flownet.pkl file not found")
+            return 1
+
+        print("RIFE model download and installation completed!")
+        return 0
+
+    except Exception as e:
+        print(f"Error: {e}")
+        return 1
+    finally:
+        print("Cleaning up temporary files...")
+
+        if zip_path.exists():
+            try:
+                zip_path.unlink()
+                print(f"Deleted: {zip_path}")
+            except Exception as e:
+                print(f"Error deleting zip file: {e}")
+
+        for item in temp_dir.iterdir():
+            if item.is_dir():
+                try:
+                    shutil.rmtree(item)
+                    print(f"Deleted directory: {item}")
+                except Exception as e:
+                    print(f"Error deleting directory {item}: {e}")
+
+        if temp_dir.exists() and not any(temp_dir.iterdir()):
+            try:
+                temp_dir.rmdir()
+                print(f"Deleted temp directory: {temp_dir}")
+            except Exception as e:
+                print(f"Error deleting temp directory: {e}")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

custom_nodes/ComfyUI-VFI/rife/model/loss.py

@@ -0,0 +1,130 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.models as models
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class EPE(nn.Module):
+    def __init__(self):
+        super(EPE, self).__init__()
+
+    def forward(self, flow, gt, loss_mask):
+        loss_map = (flow - gt.detach()) ** 2
+        loss_map = (loss_map.sum(1, True) + 1e-6) ** 0.5
+        return loss_map * loss_mask
+
+
+class Ternary(nn.Module):
+    def __init__(self):
+        super(Ternary, self).__init__()
+        patch_size = 7
+        out_channels = patch_size * patch_size
+        self.w = np.eye(out_channels).reshape((patch_size, patch_size, 1, out_channels))
+        self.w = np.transpose(self.w, (3, 2, 0, 1))
+        self.w = torch.tensor(self.w).float().to(device)
+
+    def transform(self, img):
+        patches = F.conv2d(img, self.w, padding=3, bias=None)
+        transf = patches - img
+        transf_norm = transf / torch.sqrt(0.81 + transf**2)
+        return transf_norm
+
+    def rgb2gray(self, rgb):
+        r, g, b = rgb[:, 0:1, :, :], rgb[:, 1:2, :, :], rgb[:, 2:3, :, :]
+        gray = 0.2989 * r + 0.5870 * g + 0.1140 * b
+        return gray
+
+    def hamming(self, t1, t2):
+        dist = (t1 - t2) ** 2
+        dist_norm = torch.mean(dist / (0.1 + dist), 1, True)
+        return dist_norm
+
+    def valid_mask(self, t, padding):
+        n, _, h, w = t.size()
+        inner = torch.ones(n, 1, h - 2 * padding, w - 2 * padding).type_as(t)
+        mask = F.pad(inner, [padding] * 4)
+        return mask
+
+    def forward(self, img0, img1):
+        img0 = self.transform(self.rgb2gray(img0))
+        img1 = self.transform(self.rgb2gray(img1))
+        return self.hamming(img0, img1) * self.valid_mask(img0, 1)
+
+
+class SOBEL(nn.Module):
+    def __init__(self):
+        super(SOBEL, self).__init__()
+        self.kernelX = torch.tensor(
+            [
+                [1, 0, -1],
+                [2, 0, -2],
+                [1, 0, -1],
+            ]
+        ).float()
+        self.kernelY = self.kernelX.clone().T
+        self.kernelX = self.kernelX.unsqueeze(0).unsqueeze(0).to(device)
+        self.kernelY = self.kernelY.unsqueeze(0).unsqueeze(0).to(device)
+
+    def forward(self, pred, gt):
+        N, C, H, W = pred.shape[0], pred.shape[1], pred.shape[2], pred.shape[3]
+        img_stack = torch.cat([pred.reshape(N * C, 1, H, W), gt.reshape(N * C, 1, H, W)], 0)
+        sobel_stack_x = F.conv2d(img_stack, self.kernelX, padding=1)
+        sobel_stack_y = F.conv2d(img_stack, self.kernelY, padding=1)
+        pred_X, gt_X = sobel_stack_x[: N * C], sobel_stack_x[N * C :]
+        pred_Y, gt_Y = sobel_stack_y[: N * C], sobel_stack_y[N * C :]
+
+        L1X, L1Y = torch.abs(pred_X - gt_X), torch.abs(pred_Y - gt_Y)
+        loss = L1X + L1Y
+        return loss
+
+
+class MeanShift(nn.Conv2d):
+    def __init__(self, data_mean, data_std, data_range=1, norm=True):
+        c = len(data_mean)
+        super(MeanShift, self).__init__(c, c, kernel_size=1)
+        std = torch.Tensor(data_std)
+        self.weight.data = torch.eye(c).view(c, c, 1, 1)
+        if norm:
+            self.weight.data.div_(std.view(c, 1, 1, 1))
+            self.bias.data = -1 * data_range * torch.Tensor(data_mean)  # type: ignore
+            self.bias.data.div_(std)  # type: ignore
+        else:
+            self.weight.data.mul_(std.view(c, 1, 1, 1))
+            self.bias.data = data_range * torch.Tensor(data_mean)  # type: ignore
+        self.requires_grad = False
+
+
+class VGGPerceptualLoss(torch.nn.Module):
+    def __init__(self, rank=0):
+        super(VGGPerceptualLoss, self).__init__()
+        blocks = []  # noqa: F841
+        pretrained = True
+        self.vgg_pretrained_features = models.vgg19(pretrained=pretrained).features
+        self.normalize = MeanShift([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], norm=True).cuda()
+        for param in self.parameters():
+            param.requires_grad = False
+
+    def forward(self, X, Y, indices=None):
+        X = self.normalize(X)
+        Y = self.normalize(Y)
+        indices = [2, 7, 12, 21, 30]
+        weights = [1.0 / 2.6, 1.0 / 4.8, 1.0 / 3.7, 1.0 / 5.6, 10 / 1.5]
+        k = 0
+        loss = 0
+        for i in range(indices[-1]):
+            X = self.vgg_pretrained_features[i](X)  # type: ignore
+            Y = self.vgg_pretrained_features[i](Y)  # type: ignore
+            if (i + 1) in indices:
+                loss += weights[k] * (X - Y.detach()).abs().mean() * 0.1
+                k += 1
+        return loss
+
+
+if __name__ == "__main__":
+    img0 = torch.zeros(3, 3, 256, 256).float().to(device)
+    img1 = torch.tensor(np.random.normal(0, 1, (3, 3, 256, 256))).float().to(device)
+    ternary_loss = Ternary()
+    print(ternary_loss(img0, img1).shape)

custom_nodes/ComfyUI-VFI/rife/model/pytorch_msssim/__init__.py

@@ -0,0 +1,203 @@
+from math import exp
+
+import torch
+import torch.nn.functional as F
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def gaussian(window_size, sigma):
+    gauss = torch.Tensor([exp(-((x - window_size // 2) ** 2) / float(2 * sigma**2)) for x in range(window_size)])
+    return gauss / gauss.sum()
+
+
+def create_window(window_size, channel=1):
+    _1D_window = gaussian(window_size, 1.5).unsqueeze(1)
+    _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0).to(device)
+    window = _2D_window.expand(channel, 1, window_size, window_size).contiguous()
+    return window
+
+
+def create_window_3d(window_size, channel=1):
+    _1D_window = gaussian(window_size, 1.5).unsqueeze(1)
+    _2D_window = _1D_window.mm(_1D_window.t())
+    _3D_window = _2D_window.unsqueeze(2) @ (_1D_window.t())
+    window = _3D_window.expand(1, channel, window_size, window_size, window_size).contiguous().to(device)
+    return window
+
+
+def ssim(img1, img2, window_size=11, window=None, size_average=True, full=False, val_range=None):
+    # Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh).
+    if val_range is None:
+        if torch.max(img1) > 128:
+            max_val = 255
+        else:
+            max_val = 1
+
+        if torch.min(img1) < -0.5:
+            min_val = -1
+        else:
+            min_val = 0
+        L = max_val - min_val
+    else:
+        L = val_range
+
+    padd = 0
+    (_, channel, height, width) = img1.size()
+    if window is None:
+        real_size = min(window_size, height, width)
+        window = create_window(real_size, channel=channel).to(img1.device)
+
+    # mu1 = F.conv2d(img1, window, padding=padd, groups=channel)
+    # mu2 = F.conv2d(img2, window, padding=padd, groups=channel)
+    mu1 = F.conv2d(F.pad(img1, (5, 5, 5, 5), mode="replicate"), window, padding=padd, groups=channel)
+    mu2 = F.conv2d(F.pad(img2, (5, 5, 5, 5), mode="replicate"), window, padding=padd, groups=channel)
+
+    mu1_sq = mu1.pow(2)
+    mu2_sq = mu2.pow(2)
+    mu1_mu2 = mu1 * mu2
+
+    sigma1_sq = F.conv2d(F.pad(img1 * img1, (5, 5, 5, 5), "replicate"), window, padding=padd, groups=channel) - mu1_sq
+    sigma2_sq = F.conv2d(F.pad(img2 * img2, (5, 5, 5, 5), "replicate"), window, padding=padd, groups=channel) - mu2_sq
+    sigma12 = F.conv2d(F.pad(img1 * img2, (5, 5, 5, 5), "replicate"), window, padding=padd, groups=channel) - mu1_mu2
+
+    C1 = (0.01 * L) ** 2
+    C2 = (0.03 * L) ** 2
+
+    v1 = 2.0 * sigma12 + C2
+    v2 = sigma1_sq + sigma2_sq + C2
+    cs = torch.mean(v1 / v2)  # contrast sensitivity
+
+    ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2)
+
+    if size_average:
+        ret = ssim_map.mean()
+    else:
+        ret = ssim_map.mean(1).mean(1).mean(1)
+
+    if full:
+        return ret, cs
+    return ret
+
+
+def ssim_matlab(img1, img2, window_size=11, window=None, size_average=True, full=False, val_range=None):
+    # Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh).
+    if val_range is None:
+        if torch.max(img1) > 128:
+            max_val = 255
+        else:
+            max_val = 1
+
+        if torch.min(img1) < -0.5:
+            min_val = -1
+        else:
+            min_val = 0
+        L = max_val - min_val
+    else:
+        L = val_range
+
+    padd = 0
+    (_, _, height, width) = img1.size()
+    if window is None:
+        real_size = min(window_size, height, width)
+        window = create_window_3d(real_size, channel=1).to(img1.device)
+        # Channel is set to 1 since we consider color images as volumetric images
+
+    img1 = img1.unsqueeze(1)
+    img2 = img2.unsqueeze(1)
+
+    mu1 = F.conv3d(F.pad(img1, (5, 5, 5, 5, 5, 5), mode="replicate"), window, padding=padd, groups=1)
+    mu2 = F.conv3d(F.pad(img2, (5, 5, 5, 5, 5, 5), mode="replicate"), window, padding=padd, groups=1)
+
+    mu1_sq = mu1.pow(2)
+    mu2_sq = mu2.pow(2)
+    mu1_mu2 = mu1 * mu2
+
+    sigma1_sq = F.conv3d(F.pad(img1 * img1, (5, 5, 5, 5, 5, 5), "replicate"), window, padding=padd, groups=1) - mu1_sq
+    sigma2_sq = F.conv3d(F.pad(img2 * img2, (5, 5, 5, 5, 5, 5), "replicate"), window, padding=padd, groups=1) - mu2_sq
+    sigma12 = F.conv3d(F.pad(img1 * img2, (5, 5, 5, 5, 5, 5), "replicate"), window, padding=padd, groups=1) - mu1_mu2
+
+    C1 = (0.01 * L) ** 2
+    C2 = (0.03 * L) ** 2
+
+    v1 = 2.0 * sigma12 + C2
+    v2 = sigma1_sq + sigma2_sq + C2
+    cs = torch.mean(v1 / v2)  # contrast sensitivity
+
+    ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2)
+
+    if size_average:
+        ret = ssim_map.mean()
+    else:
+        ret = ssim_map.mean(1).mean(1).mean(1)
+
+    if full:
+        return ret, cs
+    return ret
+
+
+def msssim(img1, img2, window_size=11, size_average=True, val_range=None, normalize=False):
+    device = img1.device
+    weights = torch.FloatTensor([0.0448, 0.2856, 0.3001, 0.2363, 0.1333]).to(device)
+    levels = weights.size()[0]
+    mssim = []
+    mcs = []
+    for _ in range(levels):
+        sim, cs = ssim(img1, img2, window_size=window_size, size_average=size_average, full=True, val_range=val_range)
+        mssim.append(sim)
+        mcs.append(cs)
+
+        img1 = F.avg_pool2d(img1, (2, 2))
+        img2 = F.avg_pool2d(img2, (2, 2))
+
+    mssim = torch.stack(mssim)
+    mcs = torch.stack(mcs)
+
+    # Normalize (to avoid NaNs during training unstable models, not compliant with original definition)
+    if normalize:
+        mssim = (mssim + 1) / 2
+        mcs = (mcs + 1) / 2
+
+    pow1 = mcs**weights
+    pow2 = mssim**weights
+    # From Matlab implementation https://ece.uwaterloo.ca/~z70wang/research/iwssim/
+    output = torch.prod(pow1[:-1] * pow2[-1])
+    return output
+
+
+# Classes to re-use window
+class SSIM(torch.nn.Module):
+    def __init__(self, window_size=11, size_average=True, val_range=None):
+        super(SSIM, self).__init__()
+        self.window_size = window_size
+        self.size_average = size_average
+        self.val_range = val_range
+
+        # Assume 3 channel for SSIM
+        self.channel = 3
+        self.window = create_window(window_size, channel=self.channel)
+
+    def forward(self, img1, img2):
+        (_, channel, _, _) = img1.size()
+
+        if channel == self.channel and self.window.dtype == img1.dtype:
+            window = self.window
+        else:
+            window = create_window(self.window_size, channel).to(img1.device).type(img1.dtype)
+            self.window = window
+            self.channel = channel
+
+        _ssim = ssim(img1, img2, window=window, window_size=self.window_size, size_average=self.size_average)
+        dssim = (1 - _ssim) / 2
+        return dssim
+
+
+class MSSSIM(torch.nn.Module):
+    def __init__(self, window_size=11, size_average=True, channel=3):
+        super(MSSSIM, self).__init__()
+        self.window_size = window_size
+        self.size_average = size_average
+        self.channel = channel
+
+    def forward(self, img1, img2):
+        return msssim(img1, img2, window_size=self.window_size, size_average=self.size_average)

custom_nodes/ComfyUI-VFI/rife/model/warplayer.py

@@ -0,0 +1,33 @@
+import torch
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+backwarp_tenGrid = {}
+
+
+def warp(tenInput, tenFlow):
+    k = (str(tenFlow.device), str(tenFlow.size()))
+    if k not in backwarp_tenGrid:
+        tenHorizontal = (
+            torch.linspace(-1.0, 1.0, tenFlow.shape[3], device=device)
+            .view(1, 1, 1, tenFlow.shape[3])
+            .expand(tenFlow.shape[0], -1, tenFlow.shape[2], -1)
+        )
+        tenVertical = (
+            torch.linspace(-1.0, 1.0, tenFlow.shape[2], device=device)
+            .view(1, 1, tenFlow.shape[2], 1)
+            .expand(tenFlow.shape[0], -1, -1, tenFlow.shape[3])
+        )
+        backwarp_tenGrid[k] = torch.cat([tenHorizontal, tenVertical], 1).to(device)
+
+    tenFlow = torch.cat(
+        [
+            tenFlow[:, 0:1, :, :] / ((tenInput.shape[3] - 1.0) / 2.0),
+            tenFlow[:, 1:2, :, :] / ((tenInput.shape[2] - 1.0) / 2.0),
+        ],
+        1,
+    )
+
+    g = (backwarp_tenGrid[k] + tenFlow).permute(0, 2, 3, 1)
+    return torch.nn.functional.grid_sample(
+        input=tenInput, grid=g, mode="bilinear", padding_mode="border", align_corners=True
+    )

custom_nodes/ComfyUI-VFI/rife/rife_comfyui_wrapper.py

@@ -0,0 +1,191 @@
+import os
+from typing import List, Optional, Tuple
+
+import torch
+from torch.nn import functional as F
+
+from .train_log.RIFE_HDv3 import Model
+
+
+class RIFEWrapper:
+    """Wrapper for RIFE model to work with ComfyUI Image tensors"""
+
+    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+    def __init__(self, model_path, device: Optional[torch.device] = None, use_fp16: bool = False):
+        self.device = device or torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.use_fp16 = use_fp16 and torch.cuda.is_available()
+
+        torch.set_grad_enabled(False)
+        if torch.cuda.is_available():
+            torch.backends.cudnn.enabled = True
+            torch.backends.cudnn.benchmark = True
+            if hasattr(torch.backends.cuda, "enable_mem_efficient_sdp"):
+                torch.backends.cuda.enable_mem_efficient_sdp(True)
+
+        self.model = Model()
+        self.model.load_model(model_path, -1)
+        self.model.eval()
+        self.model.device()
+
+        # Convert to fp16 if requested
+        if self.use_fp16:
+            self.model.flownet = self.model.flownet.half()
+
+    def interpolate_frames(
+        self,
+        images: torch.Tensor,
+        source_fps: float,
+        target_fps: float,
+        scale: float = 1.0,
+        progress_callback=None,
+        batch_size: int = 8,
+    ) -> torch.Tensor:
+        """
+        Interpolate frames from source FPS to target FPS
+
+        Args:
+            images: ComfyUI Image tensor [N, H, W, C] in range [0, 1]
+            source_fps: Source frame rate
+            target_fps: Target frame rate
+            scale: Scale factor for processing
+            progress_callback: Optional callback function that accepts (current, total) parameters
+            batch_size: Number of frames to process in parallel (default: 8)
+
+        Returns:
+            Interpolated ComfyUI Image tensor [M, H, W, C] in range [0, 1]
+        """
+
+        assert images.dim() == 4 and images.shape[-1] == 3, "Input must be [N, H, W, C] with C=3"
+
+        if source_fps == target_fps:
+            return images
+
+        total_source_frames = images.shape[0]
+        height, width = images.shape[1:3]
+
+        # Calculate padding
+        tmp = max(128, int(128 / scale))
+        ph = ((height - 1) // tmp + 1) * tmp
+        pw = ((width - 1) // tmp + 1) * tmp
+        padding = (0, pw - width, 0, ph - height)
+
+        # Calculate frame positions
+        frame_positions = self._calculate_target_frame_positions(source_fps, target_fps, total_source_frames)
+
+        # Pre-allocate output on CPU (NOT GPU to avoid OOM)
+        output_frames = []
+
+        # Build interpolation job list
+        interp_job_list = []
+        output_index_map = {}  # Maps job_idx -> output position
+
+        for out_idx, (source_idx1, source_idx2, interp_factor) in enumerate(frame_positions):
+            if interp_factor == 0.0 or source_idx1 == source_idx2:
+                # Direct copy, no interpolation needed
+                output_frames.append(images[source_idx1])
+            else:
+                # Need interpolation - add placeholder
+                output_frames.append(None)
+                job_idx = len(interp_job_list)
+                interp_job_list.append((source_idx1, source_idx2, interp_factor))
+                output_index_map[job_idx] = out_idx
+
+        # Process interpolations in batches with streaming
+        num_jobs = len(interp_job_list)
+        gpu_dtype = torch.float16 if self.use_fp16 else torch.float32
+
+        with torch.inference_mode():
+            for batch_start in range(0, num_jobs, batch_size):
+                batch_end = min(batch_start + batch_size, num_jobs)
+                current_batch_size = batch_end - batch_start
+
+                # Collect unique source frames needed for this batch
+                source_frames_needed = set()
+                for job_idx in range(batch_start, batch_end):
+                    source_idx1, source_idx2, _ = interp_job_list[job_idx]
+                    source_frames_needed.add(source_idx1)
+                    source_frames_needed.add(source_idx2)
+
+                # Load only required source frames to GPU
+                source_cache = {}
+                for src_idx in source_frames_needed:
+                    source_cache[src_idx] = images[src_idx].to(device=self.device, dtype=gpu_dtype)
+
+                # Prepare batch tensors on GPU
+                batch_I0 = torch.empty((current_batch_size, 3, ph, pw), dtype=gpu_dtype, device=self.device)
+                batch_I1 = torch.empty((current_batch_size, 3, ph, pw), dtype=gpu_dtype, device=self.device)
+                timesteps = []
+
+                for i, job_idx in enumerate(range(batch_start, batch_end)):
+                    source_idx1, source_idx2, interp_factor = interp_job_list[job_idx]
+
+                    # Get frames from cache (already on GPU)
+                    I0 = source_cache[source_idx1].permute(2, 0, 1).unsqueeze(0)
+                    I1 = source_cache[source_idx2].permute(2, 0, 1).unsqueeze(0)
+
+                    # Pad
+                    batch_I0[i] = F.pad(I0, padding)[0]
+                    batch_I1[i] = F.pad(I1, padding)[0]
+                    timesteps.append(interp_factor)
+
+                # Batch inference
+                interpolated_batch = self.model.inference_batch(batch_I0, batch_I1, timesteps, scale=scale)
+
+                # Transfer results to CPU and store
+                for i, job_idx in enumerate(range(batch_start, batch_end)):
+                    output_idx = output_index_map[job_idx]
+                    result = interpolated_batch[i, :, :height, :width].permute(1, 2, 0).cpu().to(torch.float32)
+                    output_frames[output_idx] = result
+
+                # Update progress
+                if progress_callback:
+                    progress_callback(batch_end, num_jobs)
+
+                # Cleanup batch memory immediately
+                del batch_I0, batch_I1, interpolated_batch, source_cache
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+
+        # Stack all output frames
+        result = torch.stack(output_frames, dim=0)
+
+        return result
+
+    def _calculate_target_frame_positions(
+        self, source_fps: float, target_fps: float, total_source_frames: int
+    ) -> List[Tuple[int, int, float]]:
+        """
+        Calculate which frames need to be generated for the target frame rate.
+
+        Returns:
+            List of (source_frame_index1, source_frame_index2, interpolation_factor) tuples
+        """
+        frame_positions = []
+
+        # Calculate the time duration of the video
+        duration = total_source_frames / source_fps
+
+        # Calculate number of target frames
+        total_target_frames = int(duration * target_fps)
+
+        for target_idx in range(total_target_frames):
+            # Calculate the time position of this target frame
+            target_time = target_idx / target_fps
+
+            # Calculate the corresponding position in source frames
+            source_position = target_time * source_fps
+
+            # Find the two source frames to interpolate between
+            source_idx1 = int(source_position)
+            source_idx2 = min(source_idx1 + 1, total_source_frames - 1)
+
+            # Calculate interpolation factor (0 means use frame1, 1 means use frame2)
+            if source_idx1 == source_idx2:
+                interpolation_factor = 0.0
+            else:
+                interpolation_factor = source_position - source_idx1
+
+            frame_positions.append((source_idx1, source_idx2, interpolation_factor))
+
+        return frame_positions

custom_nodes/ComfyUI-VFI/rife/train_log/IFNet_HDv3.py

@@ -0,0 +1,214 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from ..model.warplayer import warp
+
+# from train_log.refine import *
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
+    return nn.Sequential(
+        nn.Conv2d(
+            in_planes,
+            out_planes,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=True,
+        ),
+        nn.LeakyReLU(0.2, True),
+    )
+
+
+def conv_bn(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
+    return nn.Sequential(
+        nn.Conv2d(
+            in_planes,
+            out_planes,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=False,
+        ),
+        nn.BatchNorm2d(out_planes),
+        nn.LeakyReLU(0.2, True),
+    )
+
+
+class Head(nn.Module):
+    def __init__(self):
+        super(Head, self).__init__()
+        self.cnn0 = nn.Conv2d(3, 16, 3, 2, 1)
+        self.cnn1 = nn.Conv2d(16, 16, 3, 1, 1)
+        self.cnn2 = nn.Conv2d(16, 16, 3, 1, 1)
+        self.cnn3 = nn.ConvTranspose2d(16, 4, 4, 2, 1)
+        self.relu = nn.LeakyReLU(0.2, True)
+
+    def forward(self, x, feat=False):
+        x0 = self.cnn0(x)
+        x = self.relu(x0)
+        x1 = self.cnn1(x)
+        x = self.relu(x1)
+        x2 = self.cnn2(x)
+        x = self.relu(x2)
+        x3 = self.cnn3(x)
+        if feat:
+            return [x0, x1, x2, x3]
+        return x3
+
+
+class ResConv(nn.Module):
+    def __init__(self, c, dilation=1):
+        super(ResConv, self).__init__()
+        self.conv = nn.Conv2d(c, c, 3, 1, dilation, dilation=dilation, groups=1)
+        self.beta = nn.Parameter(torch.ones((1, c, 1, 1)), requires_grad=True)
+        self.relu = nn.LeakyReLU(0.2, True)
+
+    def forward(self, x):
+        return self.relu(self.conv(x) * self.beta + x)
+
+
+class IFBlock(nn.Module):
+    def __init__(self, in_planes, c=64):
+        super(IFBlock, self).__init__()
+        self.conv0 = nn.Sequential(
+            conv(in_planes, c // 2, 3, 2, 1),
+            conv(c // 2, c, 3, 2, 1),
+        )
+        self.convblock = nn.Sequential(
+            ResConv(c),
+            ResConv(c),
+            ResConv(c),
+            ResConv(c),
+            ResConv(c),
+            ResConv(c),
+            ResConv(c),
+            ResConv(c),
+        )
+        self.lastconv = nn.Sequential(nn.ConvTranspose2d(c, 4 * 13, 4, 2, 1), nn.PixelShuffle(2))
+
+    def forward(self, x, flow=None, scale=1):
+        x = F.interpolate(x, scale_factor=1.0 / scale, mode="bilinear", align_corners=False)
+        if flow is not None:
+            flow = F.interpolate(flow, scale_factor=1.0 / scale, mode="bilinear", align_corners=False) * 1.0 / scale
+            x = torch.cat((x, flow), 1)
+        feat = self.conv0(x)
+        feat = self.convblock(feat)
+        tmp = self.lastconv(feat)
+        tmp = F.interpolate(tmp, scale_factor=scale, mode="bilinear", align_corners=False)
+        flow = tmp[:, :4] * scale
+        mask = tmp[:, 4:5]
+        feat = tmp[:, 5:]
+        return flow, mask, feat
+
+
+class IFNet(nn.Module):
+    def __init__(self):
+        super(IFNet, self).__init__()
+        self.block0 = IFBlock(7 + 8, c=192)
+        self.block1 = IFBlock(8 + 4 + 8 + 8, c=128)
+        self.block2 = IFBlock(8 + 4 + 8 + 8, c=96)
+        self.block3 = IFBlock(8 + 4 + 8 + 8, c=64)
+        self.block4 = IFBlock(8 + 4 + 8 + 8, c=32)
+        self.encode = Head()
+
+        # not used during inference
+        """
+        self.teacher = IFBlock(8+4+8+3+8, c=64)
+        self.caltime = nn.Sequential(
+            nn.Conv2d(16+9, 8, 3, 2, 1),
+            nn.LeakyReLU(0.2, True),
+            nn.Conv2d(32, 64, 3, 2, 1),
+            nn.LeakyReLU(0.2, True),
+            nn.Conv2d(64, 64, 3, 1, 1),
+            nn.LeakyReLU(0.2, True),
+            nn.Conv2d(64, 64, 3, 1, 1),
+            nn.LeakyReLU(0.2, True),
+            nn.Conv2d(64, 1, 3, 1, 1),
+            nn.Sigmoid()
+        )
+        """
+
+    def forward(
+        self,
+        x,
+        timestep=0.5,
+        scale_list=[8, 4, 2, 1],
+        training=False,
+        fastmode=True,
+        ensemble=False,
+    ):
+        if not training:
+            channel = x.shape[1] // 2
+            img0 = x[:, :channel]
+            img1 = x[:, channel:]
+        if not torch.is_tensor(timestep):
+            timestep = (x[:, :1].clone() * 0 + 1) * timestep
+        else:
+            timestep = timestep.repeat(1, 1, img0.shape[2], img0.shape[3])
+        f0 = self.encode(img0[:, :3])
+        f1 = self.encode(img1[:, :3])
+        flow_list = []
+        merged = []
+        mask_list = []
+        warped_img0 = img0
+        warped_img1 = img1
+        flow = None
+        mask = None
+        loss_cons = 0
+        block = [self.block0, self.block1, self.block2, self.block3, self.block4]
+        for i in range(5):
+            if flow is None:
+                flow, mask, feat = block[i](
+                    torch.cat((img0[:, :3], img1[:, :3], f0, f1, timestep), 1),
+                    None,
+                    scale=scale_list[i],
+                )
+                if ensemble:
+                    print("warning: ensemble is not supported since RIFEv4.21")
+            else:
+                wf0 = warp(f0, flow[:, :2])
+                wf1 = warp(f1, flow[:, 2:4])
+                fd, m0, feat = block[i](
+                    torch.cat(
+                        (
+                            warped_img0[:, :3],
+                            warped_img1[:, :3],
+                            wf0,
+                            wf1,
+                            timestep,
+                            mask,
+                            feat,
+                        ),
+                        1,
+                    ),
+                    flow,
+                    scale=scale_list[i],
+                )
+                if ensemble:
+                    print("warning: ensemble is not supported since RIFEv4.21")
+                else:
+                    mask = m0
+                flow = flow + fd
+            mask_list.append(mask)
+            flow_list.append(flow)
+            warped_img0 = warp(img0, flow[:, :2])
+            warped_img1 = warp(img1, flow[:, 2:4])
+            merged.append((warped_img0, warped_img1))
+        mask = torch.sigmoid(mask)
+        merged[4] = warped_img0 * mask + warped_img1 * (1 - mask)
+        if not fastmode:
+            print("contextnet is removed")
+            """
+            c0 = self.contextnet(img0, flow[:, :2])
+            c1 = self.contextnet(img1, flow[:, 2:4])
+            tmp = self.unet(img0, img1, warped_img0, warped_img1, mask, flow, c0, c1)
+            res = tmp[:, :3] * 2 - 1
+            merged[4] = torch.clamp(merged[4] + res, 0, 1)
+            """
+        return flow_list, mask_list[4], merged

custom_nodes/ComfyUI-VFI/rife/train_log/RIFE_HDv3.py

@@ -0,0 +1,130 @@
+import torch
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.optim import AdamW
+
+from ..model.loss import *
+from .IFNet_HDv3 import *
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class Model:
+    def __init__(self, local_rank=-1):
+        self.flownet = IFNet()
+        self.device()
+        self.optimG = AdamW(self.flownet.parameters(), lr=1e-6, weight_decay=1e-4)
+        self.epe = EPE()
+        self.version = 4.25
+        # self.vgg = VGGPerceptualLoss().to(device)
+        self.sobel = SOBEL()
+        if local_rank != -1:
+            self.flownet = DDP(self.flownet, device_ids=[local_rank], output_device=local_rank)
+
+    def train(self):
+        self.flownet.train()
+
+    def eval(self):
+        self.flownet.eval()
+
+    def device(self):
+        self.flownet.to(device)
+
+    def load_model(self, path, rank=0):
+        def convert(param):
+            if rank == -1:
+                return {k.replace("module.", ""): v for k, v in param.items() if "module." in k}
+            else:
+                return param
+
+        if rank <= 0:
+            if torch.cuda.is_available():
+                try:
+                    # Try with weights_only=True for PyTorch >= 1.13
+                    self.flownet.load_state_dict(convert(torch.load(path, weights_only=True)), False)
+                except TypeError:
+                    # Fallback for older PyTorch versions
+                    self.flownet.load_state_dict(convert(torch.load(path)), False)
+            else:
+                try:
+                    # Try with weights_only=True for PyTorch >= 1.13
+                    self.flownet.load_state_dict(
+                        convert(torch.load(path, map_location="cpu", weights_only=True)),
+                        False,
+                    )
+                except TypeError:
+                    # Fallback for older PyTorch versions
+                    self.flownet.load_state_dict(
+                        convert(torch.load(path, map_location="cpu")),
+                        False,
+                    )
+
+    def save_model(self, path, rank=0):
+        if rank == 0:
+            torch.save(self.flownet.state_dict(), "{}/flownet.pkl".format(path))
+
+    def inference(self, img0, img1, timestep=0.5, scale=1.0):
+        imgs = torch.cat((img0, img1), 1)
+        scale_list = [16 / scale, 8 / scale, 4 / scale, 2 / scale, 1 / scale]
+        flow, mask, merged = self.flownet(imgs, timestep, scale_list)
+        # Return only the final result to save memory
+        result = merged[-1]
+        # Clear intermediate results
+        del flow, mask, merged
+        return result
+
+    def inference_batch(self, batch_img0, batch_img1, timesteps, scale=1.0):
+        """Batch inference for multiple frame pairs at once"""
+        batch_size = batch_img0.shape[0]
+
+        # Concatenate all pairs
+        imgs = torch.cat((batch_img0, batch_img1), 1)
+
+        # Pre-calculate scale list
+        scale_list = [16 / scale, 8 / scale, 4 / scale, 2 / scale, 1 / scale]
+
+        # Process all timesteps (convert list to tensor if needed)
+        if isinstance(timesteps, list):
+            timesteps = torch.tensor(timesteps, device=batch_img0.device, dtype=batch_img0.dtype)
+
+        # Batch process through network
+        results = []
+        for i in range(batch_size):
+            flow, mask, merged = self.flownet(
+                imgs[i : i + 1], timesteps[i] if timesteps.dim() > 0 else timesteps, scale_list
+            )
+            results.append(merged[-1])
+            # Clear intermediate results
+            del flow, mask, merged
+
+        # Stack results
+        result = torch.cat(results, dim=0)
+        return result
+
+    def update(self, imgs, gt, learning_rate=0, mul=1, training=True, flow_gt=None):
+        for param_group in self.optimG.param_groups:
+            param_group["lr"] = learning_rate
+        img0 = imgs[:, :3]
+        img1 = imgs[:, 3:]
+        if training:
+            self.train()
+        else:
+            self.eval()
+        scale = [16, 8, 4, 2, 1]
+        flow, mask, merged = self.flownet(torch.cat((imgs, gt), 1), scale=scale, training=training)
+        loss_l1 = (merged[-1] - gt).abs().mean()
+        loss_smooth = self.sobel(flow[-1], flow[-1] * 0).mean()
+        # loss_vgg = self.vgg(merged[-1], gt)
+        if training:
+            self.optimG.zero_grad()
+            loss_G = loss_l1 + loss_cons + loss_smooth * 0.1  # noqa: F405
+            loss_G.backward()
+            self.optimG.step()
+        else:
+            flow_teacher = flow[2]  # noqa: F841
+        return merged[-1], {
+            "mask": mask,
+            "flow": flow[-1][:, :2],
+            "loss_l1": loss_l1,
+            "loss_cons": loss_cons,  # noqa
+            "loss_smooth": loss_smooth,
+        }

custom_nodes/ComfyUI-VFI/rife/train_log/refine.py

@@ -0,0 +1,113 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from ..model.warplayer import warp
+
+
+def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
+    return nn.Sequential(
+        nn.Conv2d(
+            in_planes,
+            out_planes,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=True,
+        ),
+        nn.LeakyReLU(0.2, True),
+    )
+
+
+def conv_woact(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
+    return nn.Sequential(
+        nn.Conv2d(
+            in_planes,
+            out_planes,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=True,
+        ),
+    )
+
+
+def deconv(in_planes, out_planes, kernel_size=4, stride=2, padding=1):
+    return nn.Sequential(
+        torch.nn.ConvTranspose2d(
+            in_channels=in_planes,
+            out_channels=out_planes,
+            kernel_size=4,
+            stride=2,
+            padding=1,
+            bias=True,
+        ),
+        nn.LeakyReLU(0.2, True),
+    )
+
+
+class Conv2(nn.Module):
+    def __init__(self, in_planes, out_planes, stride=2):
+        super(Conv2, self).__init__()
+        self.conv1 = conv(in_planes, out_planes, 3, stride, 1)
+        self.conv2 = conv(out_planes, out_planes, 3, 1, 1)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.conv2(x)
+        return x
+
+
+c = 16
+
+
+class Contextnet(nn.Module):
+    def __init__(self):
+        super(Contextnet, self).__init__()
+        self.conv1 = Conv2(3, c)
+        self.conv2 = Conv2(c, 2 * c)
+        self.conv3 = Conv2(2 * c, 4 * c)
+        self.conv4 = Conv2(4 * c, 8 * c)
+
+    def forward(self, x, flow):
+        x = self.conv1(x)
+        flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5
+        f1 = warp(x, flow)
+        x = self.conv2(x)
+        flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5
+        f2 = warp(x, flow)
+        x = self.conv3(x)
+        flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5
+        f3 = warp(x, flow)
+        x = self.conv4(x)
+        flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False) * 0.5
+        f4 = warp(x, flow)
+        return [f1, f2, f3, f4]
+
+
+class Unet(nn.Module):
+    def __init__(self):
+        super(Unet, self).__init__()
+        self.down0 = Conv2(17, 2 * c)
+        self.down1 = Conv2(4 * c, 4 * c)
+        self.down2 = Conv2(8 * c, 8 * c)
+        self.down3 = Conv2(16 * c, 16 * c)
+        self.up0 = deconv(32 * c, 8 * c)
+        self.up1 = deconv(16 * c, 4 * c)
+        self.up2 = deconv(8 * c, 2 * c)
+        self.up3 = deconv(4 * c, c)
+        self.conv = nn.Conv2d(c, 3, 3, 1, 1)
+
+    def forward(self, img0, img1, warped_img0, warped_img1, mask, flow, c0, c1):
+        s0 = self.down0(torch.cat((img0, img1, warped_img0, warped_img1, mask, flow), 1))
+        s1 = self.down1(torch.cat((s0, c0[0], c1[0]), 1))
+        s2 = self.down2(torch.cat((s1, c0[1], c1[1]), 1))
+        s3 = self.down3(torch.cat((s2, c0[2], c1[2]), 1))
+        x = self.up0(torch.cat((s3, c0[3], c1[3]), 1))
+        x = self.up1(torch.cat((x, s2), 1))
+        x = self.up2(torch.cat((x, s1), 1))
+        x = self.up3(torch.cat((x, s0), 1))
+        x = self.conv(x)
+        return torch.sigmoid(x)