Buckets:
| """ | |
| LipSync Training Dataset | |
| ========================== | |
| PyTorch Dataset for loading preprocessed face frames, cached latents, | |
| audio features, and mel spectrograms for training. | |
| Handles: | |
| - Frame pair sampling (target + reference from same identity) | |
| - Audio-visual alignment | |
| - Data augmentation | |
| - Support for both cached latents and raw images | |
| """ | |
| import json | |
| import random | |
| from pathlib import Path | |
| from typing import Optional | |
| import cv2 | |
| import librosa | |
| import numpy as np | |
| import torch | |
| from PIL import Image | |
| from torch.utils.data import Dataset | |
| from torchvision import transforms | |
| class LipSyncDataset(Dataset): | |
| """Dataset for LipSync model training. | |
| Each sample returns: | |
| - target_latent: VAE latent of the target (ground truth) frame | |
| - reference_latent: VAE latent of a random frame from same identity | |
| - audio_features: Whisper features aligned to the target frame | |
| - mel_segment: Mel spectrogram segment for SyncNet | |
| - target_image: Original face image (for perceptual loss) | |
| - lip_image: Lip crop (for SyncNet loss) | |
| """ | |
| def __init__( | |
| self, | |
| processed_dir: str, | |
| cache_dir: str, | |
| image_size: int = 256, | |
| fps: int = 25, | |
| mel_step_size: int = 16, | |
| num_audio_frames: int = 5, | |
| sample_rate: int = 16000, | |
| augment: bool = True, | |
| max_videos: Optional[int] = None, | |
| ): | |
| self.processed_dir = Path(processed_dir) | |
| self.cache_dir = Path(cache_dir) | |
| self.image_size = image_size | |
| self.fps = fps | |
| self.mel_step_size = mel_step_size | |
| self.num_audio_frames = num_audio_frames | |
| self.sample_rate = sample_rate | |
| self.augment = augment | |
| # Image transforms | |
| self.transform = transforms.Compose([ | |
| transforms.Resize(image_size), | |
| transforms.CenterCrop(image_size), | |
| transforms.ToTensor(), | |
| transforms.Normalize([0.5], [0.5]), # Scale to [-1, 1] | |
| ]) | |
| # Augmentation transforms (only during training) | |
| self.aug_transform = transforms.Compose([ | |
| transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1), | |
| transforms.RandomHorizontalFlip(p=0.5), | |
| ]) if augment else None | |
| # Load all video metadata and build sample index | |
| self.samples = [] | |
| self._build_index(max_videos) | |
| def _build_index(self, max_videos: Optional[int] = None): | |
| """Build a flat index of (video_name, frame_idx) pairs.""" | |
| # Find all processed video directories | |
| if not self.processed_dir.exists(): | |
| print(f"Warning: processed_dir {self.processed_dir} does not exist") | |
| return | |
| video_dirs = sorted([ | |
| d for d in self.processed_dir.iterdir() | |
| if d.is_dir() and (d / "metadata.json").exists() | |
| ]) | |
| if max_videos: | |
| video_dirs = video_dirs[:max_videos] | |
| for video_dir in video_dirs: | |
| video_name = video_dir.name | |
| meta_path = video_dir / "metadata.json" | |
| with open(meta_path, "r") as f: | |
| metadata = json.load(f) | |
| n_frames = metadata["total_frames"] | |
| # Each frame (with enough context) becomes a trainable sample | |
| # Need num_audio_frames for temporal context | |
| half_window = self.num_audio_frames // 2 | |
| for frame_idx in range(half_window, n_frames - half_window): | |
| self.samples.append({ | |
| "video_name": video_name, | |
| "frame_idx": frame_idx, | |
| "total_frames": n_frames, | |
| }) | |
| print(f"Dataset: {len(self.samples)} samples from {len(video_dirs)} videos") | |
| def _load_frame(self, video_name: str, frame_idx: int) -> Optional[torch.Tensor]: | |
| """Load a face frame as tensor.""" | |
| frame_path = self.processed_dir / video_name / "frames" / f"{frame_idx:06d}.png" | |
| if not frame_path.exists(): | |
| return None | |
| img = Image.open(frame_path).convert("RGB") | |
| return self.transform(img) | |
| def _load_lip_frame(self, video_name: str, frame_idx: int) -> Optional[torch.Tensor]: | |
| """Load a lip crop for SyncNet.""" | |
| lip_path = self.processed_dir / video_name / "lips" / f"{frame_idx:06d}.png" | |
| if not lip_path.exists(): | |
| return None | |
| img = Image.open(lip_path).convert("RGB") | |
| lip_transform = transforms.Compose([ | |
| transforms.Resize(96), | |
| transforms.CenterCrop(96), | |
| transforms.ToTensor(), | |
| transforms.Normalize([0.5], [0.5]), | |
| ]) | |
| return lip_transform(img) | |
| def _load_cached_latent(self, video_name: str, frame_idx: int) -> Optional[torch.Tensor]: | |
| """Load pre-computed VAE latent for a frame.""" | |
| latents_path = self.cache_dir / video_name / "vae_latents.npy" | |
| if not latents_path.exists(): | |
| return None | |
| latents = np.load(str(latents_path), mmap_mode="r") | |
| if frame_idx >= len(latents): | |
| return None | |
| return torch.from_numpy(latents[frame_idx].copy()).float() | |
| def _load_audio_features( | |
| self, video_name: str, frame_idx: int | |
| ) -> Optional[torch.Tensor]: | |
| """Load Whisper audio features for frames around frame_idx. | |
| Returns a fixed-size tensor (num_audio_tokens, D) to allow batching. | |
| """ | |
| features_path = self.cache_dir / video_name / "whisper_features.npy" | |
| if not features_path.exists(): | |
| return None | |
| features = np.load(str(features_path)) # (1, T, D) | |
| if features.ndim == 3: | |
| features = features[0] # Remove batch dim -> (T, D) | |
| feat_dim = features.shape[-1] | |
| # Fixed number of audio tokens to return (for consistent batching) | |
| target_len = self.num_audio_frames * 4 # e.g. 5 * 4 = 20 tokens | |
| # Map video frame to Whisper feature index | |
| whisper_fps = 1500 / 30 # 50 features per second | |
| video_fps = self.fps | |
| center_whisper = int(frame_idx * whisper_fps / video_fps) | |
| half_window = target_len // 2 | |
| start = max(0, center_whisper - half_window) | |
| end = min(len(features), center_whisper + half_window) | |
| audio_segment = features[start:end] | |
| # Pad or crop to fixed target_len for consistent batching | |
| result = np.zeros((target_len, feat_dim), dtype=np.float32) | |
| if len(audio_segment) > 0: | |
| n = min(len(audio_segment), target_len) | |
| result[:n] = audio_segment[:n] | |
| return torch.from_numpy(result).float() | |
| def _load_mel_segment( | |
| self, video_name: str, frame_idx: int | |
| ) -> Optional[torch.Tensor]: | |
| """Load mel spectrogram segment aligned to frame.""" | |
| mel_path = self.cache_dir / video_name / "mel_spectrogram.npy" | |
| if not mel_path.exists(): | |
| return None | |
| mel = np.load(str(mel_path)) # (n_mels, T) | |
| # Map video frame to mel frame | |
| # hop_length=256, sr=16000 -> mel_fps = 16000/256 = 62.5 | |
| mel_fps = self.sample_rate / 256 | |
| mel_per_video_frame = mel_fps / self.fps | |
| center_mel = int(frame_idx * mel_per_video_frame) | |
| half_window = self.num_audio_frames * self.mel_step_size // 2 | |
| start = max(0, center_mel - half_window) | |
| end = min(mel.shape[1], center_mel + half_window) | |
| mel_segment = mel[:, start:end] | |
| # Pad if necessary | |
| target_width = self.num_audio_frames * self.mel_step_size | |
| if mel_segment.shape[1] < target_width: | |
| pad_width = target_width - mel_segment.shape[1] | |
| mel_segment = np.pad(mel_segment, ((0, 0), (0, pad_width))) | |
| mel_segment = mel_segment[:, :target_width] # Crop to exact size | |
| return torch.from_numpy(mel_segment.copy()).float().unsqueeze(0) # (1, n_mels, T) | |
| def __len__(self) -> int: | |
| return len(self.samples) | |
| def __getitem__(self, idx: int) -> dict: | |
| sample = self.samples[idx] | |
| video_name = sample["video_name"] | |
| frame_idx = sample["frame_idx"] | |
| total_frames = sample["total_frames"] | |
| # Pick a reference frame (random frame from same video, not too close) | |
| ref_candidates = list(range(0, max(0, frame_idx - 10))) + \ | |
| list(range(min(total_frames, frame_idx + 10), total_frames)) | |
| if not ref_candidates: | |
| ref_candidates = list(range(total_frames)) | |
| ref_idx = random.choice(ref_candidates) | |
| # Load data - prefer cached latents, fallback to images | |
| target_latent = self._load_cached_latent(video_name, frame_idx) | |
| ref_latent = self._load_cached_latent(video_name, ref_idx) | |
| # Always load images too (for perceptual loss / visualization) | |
| target_image = self._load_frame(video_name, frame_idx) | |
| ref_image = self._load_frame(video_name, ref_idx) | |
| # Load audio features | |
| audio_features = self._load_audio_features(video_name, frame_idx) | |
| # Load mel for SyncNet | |
| mel_segment = self._load_mel_segment(video_name, frame_idx) | |
| # Load lip crop for SyncNet | |
| lip_frames = [] | |
| half_w = self.num_audio_frames // 2 | |
| for i in range(frame_idx - half_w, frame_idx + half_w + 1): | |
| li = max(0, min(total_frames - 1, i)) | |
| lip = self._load_lip_frame(video_name, li) | |
| if lip is not None: | |
| lip_frames.append(lip) | |
| # Pad lip frames if needed | |
| while len(lip_frames) < self.num_audio_frames: | |
| if lip_frames: | |
| lip_frames.append(lip_frames[-1]) | |
| else: | |
| lip_frames.append(torch.zeros(3, 96, 96)) | |
| lip_frames = lip_frames[:self.num_audio_frames] | |
| lip_stack = torch.stack(lip_frames) # (T, C, H, W) | |
| # Handle missing data with zeros | |
| if target_latent is None: | |
| target_latent = torch.zeros(4, 32, 32) | |
| if ref_latent is None: | |
| ref_latent = torch.zeros(4, 32, 32) | |
| if target_image is None: | |
| target_image = torch.zeros(3, self.image_size, self.image_size) | |
| if ref_image is None: | |
| ref_image = torch.zeros(3, self.image_size, self.image_size) | |
| if audio_features is None: | |
| audio_features = torch.zeros(10, 384) | |
| if mel_segment is None: | |
| mel_segment = torch.zeros(1, 80, self.num_audio_frames * self.mel_step_size) | |
| return { | |
| "target_latent": target_latent, | |
| "reference_latent": ref_latent, | |
| "target_image": target_image, | |
| "reference_image": ref_image, | |
| "audio_features": audio_features, | |
| "mel_segment": mel_segment, | |
| "lip_frames": lip_stack, | |
| "video_name": video_name, | |
| "frame_idx": frame_idx, | |
| } | |
| class SyncNetDataset(Dataset): | |
| """Simplified dataset for SyncNet training. | |
| Returns lip frame sequences paired with mel spectrograms. | |
| Handles positive (in-sync) and negative (out-of-sync) pairs. | |
| """ | |
| def __init__( | |
| self, | |
| processed_dir: str, | |
| cache_dir: str = "", | |
| image_size: int = 96, | |
| fps: int = 25, | |
| num_frames: int = 5, | |
| mel_step_size: int = 16, | |
| sample_rate: int = 16000, | |
| max_videos: Optional[int] = None, | |
| ): | |
| self.processed_dir = Path(processed_dir) | |
| self.cache_dir = Path(cache_dir) if cache_dir else self.processed_dir.parent / "cache" | |
| self.image_size = image_size | |
| self.fps = fps | |
| self.num_frames = num_frames | |
| self.mel_step_size = mel_step_size | |
| self.sample_rate = sample_rate | |
| self.transform = transforms.Compose([ | |
| transforms.Resize(image_size), | |
| transforms.CenterCrop(image_size), | |
| transforms.ToTensor(), | |
| transforms.Normalize([0.5], [0.5]), | |
| ]) | |
| self.samples = [] | |
| self._build_index(max_videos) | |
| def _build_index(self, max_videos): | |
| if not self.processed_dir.exists(): | |
| return | |
| video_dirs = sorted([ | |
| d for d in self.processed_dir.iterdir() | |
| if d.is_dir() and (d / "metadata.json").exists() | |
| ]) | |
| if max_videos: | |
| video_dirs = video_dirs[:max_videos] | |
| for video_dir in video_dirs: | |
| video_name = video_dir.name | |
| meta_path = video_dir / "metadata.json" | |
| with open(meta_path) as f: | |
| metadata = json.load(f) | |
| n_frames = metadata["total_frames"] | |
| half_w = self.num_frames // 2 | |
| for idx in range(half_w, n_frames - half_w): | |
| self.samples.append({ | |
| "video_name": video_name, | |
| "frame_idx": idx, | |
| "total_frames": n_frames, | |
| }) | |
| print(f"SyncNet Dataset: {len(self.samples)} samples") | |
| def __len__(self): | |
| return len(self.samples) | |
| def __getitem__(self, idx): | |
| sample = self.samples[idx] | |
| video_name = sample["video_name"] | |
| frame_idx = sample["frame_idx"] | |
| total_frames = sample["total_frames"] | |
| # Load lip frames | |
| half_w = self.num_frames // 2 | |
| lip_frames = [] | |
| for i in range(frame_idx - half_w, frame_idx + half_w + 1): | |
| fi = max(0, min(total_frames - 1, i)) | |
| lip_path = self.processed_dir / video_name / "lips" / f"{fi:06d}.png" | |
| if lip_path.exists(): | |
| img = Image.open(lip_path).convert("RGB") | |
| lip_frames.append(self.transform(img)) | |
| else: | |
| lip_frames.append(torch.zeros(3, self.image_size, self.image_size)) | |
| lip_frames = lip_frames[:self.num_frames] | |
| while len(lip_frames) < self.num_frames: | |
| lip_frames.append(lip_frames[-1] if lip_frames else torch.zeros(3, self.image_size, self.image_size)) | |
| lip_stack = torch.stack(lip_frames) # (T, C, H, W) | |
| # Load mel spectrogram | |
| mel_path = self.cache_dir / video_name / "mel_spectrogram.npy" | |
| if mel_path.exists(): | |
| mel = np.load(str(mel_path)) | |
| mel_fps = self.sample_rate / 256 | |
| mel_per_frame = mel_fps / self.fps | |
| center = int(frame_idx * mel_per_frame) | |
| half_mel = self.num_frames * self.mel_step_size // 2 | |
| start = max(0, center - half_mel) | |
| end = min(mel.shape[1], center + half_mel) | |
| mel_seg = mel[:, start:end] | |
| target_w = self.num_frames * self.mel_step_size | |
| if mel_seg.shape[1] < target_w: | |
| mel_seg = np.pad(mel_seg, ((0, 0), (0, target_w - mel_seg.shape[1]))) | |
| mel_seg = mel_seg[:, :target_w] | |
| mel_tensor = torch.from_numpy(mel_seg.copy()).float().unsqueeze(0) | |
| else: | |
| mel_tensor = torch.zeros(1, 80, self.num_frames * self.mel_step_size) | |
| # 50% chance positive (in-sync), 50% chance negative (out-of-sync) | |
| is_positive = random.random() > 0.5 | |
| if not is_positive: | |
| # Get a random mel from a different frame (out of sync) | |
| neg_idx = random.randint(0, len(self.samples) - 1) | |
| neg_sample = self.samples[neg_idx] | |
| neg_mel_path = self.cache_dir / neg_sample["video_name"] / "mel_spectrogram.npy" | |
| if neg_mel_path.exists(): | |
| neg_mel = np.load(str(neg_mel_path)) | |
| neg_center = int(neg_sample["frame_idx"] * mel_per_frame) | |
| ns = max(0, neg_center - half_mel) | |
| ne = min(neg_mel.shape[1], neg_center + half_mel) | |
| neg_seg = neg_mel[:, ns:ne] | |
| if neg_seg.shape[1] < target_w: | |
| neg_seg = np.pad(neg_seg, ((0, 0), (0, target_w - neg_seg.shape[1]))) | |
| neg_seg = neg_seg[:, :target_w] | |
| mel_tensor = torch.from_numpy(neg_seg.copy()).float().unsqueeze(0) | |
| label = 1.0 if is_positive else 0.0 | |
| return { | |
| "lip_frames": lip_stack, | |
| "mel": mel_tensor, | |
| "label": torch.tensor(label), | |
| } | |
Xet Storage Details
- Size:
- 16.5 kB
- Xet hash:
- 03e9d0f0d196d2b767eba7e243f80856e96d0511c4e6980c9de0ac44373c1f9f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.