import os import random import torch from torch.utils.data import Dataset from decord import VideoReader, cpu import pandas as pd class DatasetVideoLoader(Dataset): """ Dataset for loading videos and captions from a CSV file. CSV file contains two columns: 'path' and 'text', where: - 'path' is the path to the video file - 'text' is the caption for the video. """ def __init__( self, csv_file, resolution, video_length, frame_stride=4, subset_split="all", clip_length=1.0, random_stride=False, mode="video", ): self.csv_file = csv_file self.resolution = resolution self.video_length = video_length self.subset_split = subset_split self.frame_stride = frame_stride self.clip_length = clip_length self.random_stride = random_stride self.mode = mode assert self.subset_split in ["train", "test", "val", "all"] self.exts = ["avi", "mp4", "webm"] if isinstance(self.resolution, int): self.resolution = [self.resolution, self.resolution] # Load dataset from CSV file self._make_dataset() def _make_dataset(self): """ Load video paths and captions from the CSV file. """ self.videos = pd.read_csv(self.csv_file) print(f"Loaded {len(self.videos)} videos from {self.csv_file}") if self.subset_split == "val": self.videos = self.videos[-300:] elif self.subset_split == "train": self.videos = self.videos[:-300] elif self.subset_split == "test": self.videos = self.videos[-30:] print(f"Number of videos = {len(self.videos)}") # Create video indices for image mode self.video_indices = list(range(len(self.videos))) def set_mode(self, mode): self.mode = mode def _get_video_path(self, index): return self.videos.iloc[index]["path"] def __getitem__(self, index): if self.mode == "image": return self.__getitem__images(index) else: return self.__getitem__video(index) def __getitem__video(self, index): while True: video_path = self.videos.iloc[index]["path"] caption = self.videos.iloc[index]["text"] try: video_reader = VideoReader( video_path, ctx=cpu(0), width=self.resolution[1], height=self.resolution[0], ) if len(video_reader) < self.video_length: index = (index + 1) % len(self.videos) continue else: break except Exception as e: print(f"Load video failed! path = {video_path}, error: {str(e)}") index = (index + 1) % len(self.videos) continue if self.random_stride: self.frame_stride = random.choice([4, 8, 12, 16]) all_frames = list(range(0, len(video_reader), self.frame_stride)) if len(all_frames) < self.video_length: all_frames = list(range(0, len(video_reader), 1)) # Select random clip rand_idx = random.randint(0, len(all_frames) - self.video_length) frame_indices = all_frames[rand_idx : rand_idx + self.video_length] frames = video_reader.get_batch(frame_indices) assert ( frames.shape[0] == self.video_length ), f"{len(frames)}, self.video_length={self.video_length}" frames = ( torch.tensor(frames.asnumpy()).permute(3, 0, 1, 2).float() ) # [t,h,w,c] -> [c,t,h,w] assert ( frames.shape[2] == self.resolution[0] and frames.shape[3] == self.resolution[1] ), f"frames={frames.shape}, self.resolution={self.resolution}" frames = (frames / 255 - 0.5) * 2 return {"video": frames, "caption": caption, "is_video": True} def __getitem__images(self, index): frames_list = [] for i in range(self.video_length): # Get a unique video for each frame video_index = (index + i) % len(self.video_indices) video_path = self._get_video_path(video_index) try: video_reader = VideoReader( video_path, ctx=cpu(0), width=self.resolution[1], height=self.resolution[0], ) except Exception as e: print(f"Load video failed! path = {video_path}, error = {e}") # Skip this video and try the next one return self.__getitem__images((index + 1) % len(self.video_indices)) # Randomly select a frame from the video rand_idx = random.randint(0, len(video_reader) - 1) frame = video_reader[rand_idx] frame_tensor = ( torch.tensor(frame.asnumpy()).permute(2, 0, 1).float().unsqueeze(0) ) # [h,w,c] -> [c,h,w] -> [1, c, h, w] frames_list.append(frame_tensor) frames = torch.cat(frames_list, dim=0) frames = (frames / 255 - 0.5) * 2 frames = frames.permute(1, 0, 2, 3) assert ( frames.shape[2] == self.resolution[0] and frames.shape[3] == self.resolution[1] ), f"frame={frames.shape}, self.resolution={self.resolution}" data = {"video": frames, "is_video": False} return data def __len__(self): return len(self.videos)