minhluanquach's picture
download
raw
16.5 kB
"""
LipSync Training Dataset
==========================
PyTorch Dataset for loading preprocessed face frames, cached latents,
audio features, and mel spectrograms for training.
Handles:
- Frame pair sampling (target + reference from same identity)
- Audio-visual alignment
- Data augmentation
- Support for both cached latents and raw images
"""
import json
import random
from pathlib import Path
from typing import Optional
import cv2
import librosa
import numpy as np
import torch
from PIL import Image
from torch.utils.data import Dataset
from torchvision import transforms
class LipSyncDataset(Dataset):
"""Dataset for LipSync model training.
Each sample returns:
- target_latent: VAE latent of the target (ground truth) frame
- reference_latent: VAE latent of a random frame from same identity
- audio_features: Whisper features aligned to the target frame
- mel_segment: Mel spectrogram segment for SyncNet
- target_image: Original face image (for perceptual loss)
- lip_image: Lip crop (for SyncNet loss)
"""
def __init__(
self,
processed_dir: str,
cache_dir: str,
image_size: int = 256,
fps: int = 25,
mel_step_size: int = 16,
num_audio_frames: int = 5,
sample_rate: int = 16000,
augment: bool = True,
max_videos: Optional[int] = None,
):
self.processed_dir = Path(processed_dir)
self.cache_dir = Path(cache_dir)
self.image_size = image_size
self.fps = fps
self.mel_step_size = mel_step_size
self.num_audio_frames = num_audio_frames
self.sample_rate = sample_rate
self.augment = augment
# Image transforms
self.transform = transforms.Compose([
transforms.Resize(image_size),
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]), # Scale to [-1, 1]
])
# Augmentation transforms (only during training)
self.aug_transform = transforms.Compose([
transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1),
transforms.RandomHorizontalFlip(p=0.5),
]) if augment else None
# Load all video metadata and build sample index
self.samples = []
self._build_index(max_videos)
def _build_index(self, max_videos: Optional[int] = None):
"""Build a flat index of (video_name, frame_idx) pairs."""
# Find all processed video directories
if not self.processed_dir.exists():
print(f"Warning: processed_dir {self.processed_dir} does not exist")
return
video_dirs = sorted([
d for d in self.processed_dir.iterdir()
if d.is_dir() and (d / "metadata.json").exists()
])
if max_videos:
video_dirs = video_dirs[:max_videos]
for video_dir in video_dirs:
video_name = video_dir.name
meta_path = video_dir / "metadata.json"
with open(meta_path, "r") as f:
metadata = json.load(f)
n_frames = metadata["total_frames"]
# Each frame (with enough context) becomes a trainable sample
# Need num_audio_frames for temporal context
half_window = self.num_audio_frames // 2
for frame_idx in range(half_window, n_frames - half_window):
self.samples.append({
"video_name": video_name,
"frame_idx": frame_idx,
"total_frames": n_frames,
})
print(f"Dataset: {len(self.samples)} samples from {len(video_dirs)} videos")
def _load_frame(self, video_name: str, frame_idx: int) -> Optional[torch.Tensor]:
"""Load a face frame as tensor."""
frame_path = self.processed_dir / video_name / "frames" / f"{frame_idx:06d}.png"
if not frame_path.exists():
return None
img = Image.open(frame_path).convert("RGB")
return self.transform(img)
def _load_lip_frame(self, video_name: str, frame_idx: int) -> Optional[torch.Tensor]:
"""Load a lip crop for SyncNet."""
lip_path = self.processed_dir / video_name / "lips" / f"{frame_idx:06d}.png"
if not lip_path.exists():
return None
img = Image.open(lip_path).convert("RGB")
lip_transform = transforms.Compose([
transforms.Resize(96),
transforms.CenterCrop(96),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
])
return lip_transform(img)
def _load_cached_latent(self, video_name: str, frame_idx: int) -> Optional[torch.Tensor]:
"""Load pre-computed VAE latent for a frame."""
latents_path = self.cache_dir / video_name / "vae_latents.npy"
if not latents_path.exists():
return None
latents = np.load(str(latents_path), mmap_mode="r")
if frame_idx >= len(latents):
return None
return torch.from_numpy(latents[frame_idx].copy()).float()
def _load_audio_features(
self, video_name: str, frame_idx: int
) -> Optional[torch.Tensor]:
"""Load Whisper audio features for frames around frame_idx.
Returns a fixed-size tensor (num_audio_tokens, D) to allow batching.
"""
features_path = self.cache_dir / video_name / "whisper_features.npy"
if not features_path.exists():
return None
features = np.load(str(features_path)) # (1, T, D)
if features.ndim == 3:
features = features[0] # Remove batch dim -> (T, D)
feat_dim = features.shape[-1]
# Fixed number of audio tokens to return (for consistent batching)
target_len = self.num_audio_frames * 4 # e.g. 5 * 4 = 20 tokens
# Map video frame to Whisper feature index
whisper_fps = 1500 / 30 # 50 features per second
video_fps = self.fps
center_whisper = int(frame_idx * whisper_fps / video_fps)
half_window = target_len // 2
start = max(0, center_whisper - half_window)
end = min(len(features), center_whisper + half_window)
audio_segment = features[start:end]
# Pad or crop to fixed target_len for consistent batching
result = np.zeros((target_len, feat_dim), dtype=np.float32)
if len(audio_segment) > 0:
n = min(len(audio_segment), target_len)
result[:n] = audio_segment[:n]
return torch.from_numpy(result).float()
def _load_mel_segment(
self, video_name: str, frame_idx: int
) -> Optional[torch.Tensor]:
"""Load mel spectrogram segment aligned to frame."""
mel_path = self.cache_dir / video_name / "mel_spectrogram.npy"
if not mel_path.exists():
return None
mel = np.load(str(mel_path)) # (n_mels, T)
# Map video frame to mel frame
# hop_length=256, sr=16000 -> mel_fps = 16000/256 = 62.5
mel_fps = self.sample_rate / 256
mel_per_video_frame = mel_fps / self.fps
center_mel = int(frame_idx * mel_per_video_frame)
half_window = self.num_audio_frames * self.mel_step_size // 2
start = max(0, center_mel - half_window)
end = min(mel.shape[1], center_mel + half_window)
mel_segment = mel[:, start:end]
# Pad if necessary
target_width = self.num_audio_frames * self.mel_step_size
if mel_segment.shape[1] < target_width:
pad_width = target_width - mel_segment.shape[1]
mel_segment = np.pad(mel_segment, ((0, 0), (0, pad_width)))
mel_segment = mel_segment[:, :target_width] # Crop to exact size
return torch.from_numpy(mel_segment.copy()).float().unsqueeze(0) # (1, n_mels, T)
def __len__(self) -> int:
return len(self.samples)
def __getitem__(self, idx: int) -> dict:
sample = self.samples[idx]
video_name = sample["video_name"]
frame_idx = sample["frame_idx"]
total_frames = sample["total_frames"]
# Pick a reference frame (random frame from same video, not too close)
ref_candidates = list(range(0, max(0, frame_idx - 10))) + \
list(range(min(total_frames, frame_idx + 10), total_frames))
if not ref_candidates:
ref_candidates = list(range(total_frames))
ref_idx = random.choice(ref_candidates)
# Load data - prefer cached latents, fallback to images
target_latent = self._load_cached_latent(video_name, frame_idx)
ref_latent = self._load_cached_latent(video_name, ref_idx)
# Always load images too (for perceptual loss / visualization)
target_image = self._load_frame(video_name, frame_idx)
ref_image = self._load_frame(video_name, ref_idx)
# Load audio features
audio_features = self._load_audio_features(video_name, frame_idx)
# Load mel for SyncNet
mel_segment = self._load_mel_segment(video_name, frame_idx)
# Load lip crop for SyncNet
lip_frames = []
half_w = self.num_audio_frames // 2
for i in range(frame_idx - half_w, frame_idx + half_w + 1):
li = max(0, min(total_frames - 1, i))
lip = self._load_lip_frame(video_name, li)
if lip is not None:
lip_frames.append(lip)
# Pad lip frames if needed
while len(lip_frames) < self.num_audio_frames:
if lip_frames:
lip_frames.append(lip_frames[-1])
else:
lip_frames.append(torch.zeros(3, 96, 96))
lip_frames = lip_frames[:self.num_audio_frames]
lip_stack = torch.stack(lip_frames) # (T, C, H, W)
# Handle missing data with zeros
if target_latent is None:
target_latent = torch.zeros(4, 32, 32)
if ref_latent is None:
ref_latent = torch.zeros(4, 32, 32)
if target_image is None:
target_image = torch.zeros(3, self.image_size, self.image_size)
if ref_image is None:
ref_image = torch.zeros(3, self.image_size, self.image_size)
if audio_features is None:
audio_features = torch.zeros(10, 384)
if mel_segment is None:
mel_segment = torch.zeros(1, 80, self.num_audio_frames * self.mel_step_size)
return {
"target_latent": target_latent,
"reference_latent": ref_latent,
"target_image": target_image,
"reference_image": ref_image,
"audio_features": audio_features,
"mel_segment": mel_segment,
"lip_frames": lip_stack,
"video_name": video_name,
"frame_idx": frame_idx,
}
class SyncNetDataset(Dataset):
"""Simplified dataset for SyncNet training.
Returns lip frame sequences paired with mel spectrograms.
Handles positive (in-sync) and negative (out-of-sync) pairs.
"""
def __init__(
self,
processed_dir: str,
cache_dir: str = "",
image_size: int = 96,
fps: int = 25,
num_frames: int = 5,
mel_step_size: int = 16,
sample_rate: int = 16000,
max_videos: Optional[int] = None,
):
self.processed_dir = Path(processed_dir)
self.cache_dir = Path(cache_dir) if cache_dir else self.processed_dir.parent / "cache"
self.image_size = image_size
self.fps = fps
self.num_frames = num_frames
self.mel_step_size = mel_step_size
self.sample_rate = sample_rate
self.transform = transforms.Compose([
transforms.Resize(image_size),
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
])
self.samples = []
self._build_index(max_videos)
def _build_index(self, max_videos):
if not self.processed_dir.exists():
return
video_dirs = sorted([
d for d in self.processed_dir.iterdir()
if d.is_dir() and (d / "metadata.json").exists()
])
if max_videos:
video_dirs = video_dirs[:max_videos]
for video_dir in video_dirs:
video_name = video_dir.name
meta_path = video_dir / "metadata.json"
with open(meta_path) as f:
metadata = json.load(f)
n_frames = metadata["total_frames"]
half_w = self.num_frames // 2
for idx in range(half_w, n_frames - half_w):
self.samples.append({
"video_name": video_name,
"frame_idx": idx,
"total_frames": n_frames,
})
print(f"SyncNet Dataset: {len(self.samples)} samples")
def __len__(self):
return len(self.samples)
def __getitem__(self, idx):
sample = self.samples[idx]
video_name = sample["video_name"]
frame_idx = sample["frame_idx"]
total_frames = sample["total_frames"]
# Load lip frames
half_w = self.num_frames // 2
lip_frames = []
for i in range(frame_idx - half_w, frame_idx + half_w + 1):
fi = max(0, min(total_frames - 1, i))
lip_path = self.processed_dir / video_name / "lips" / f"{fi:06d}.png"
if lip_path.exists():
img = Image.open(lip_path).convert("RGB")
lip_frames.append(self.transform(img))
else:
lip_frames.append(torch.zeros(3, self.image_size, self.image_size))
lip_frames = lip_frames[:self.num_frames]
while len(lip_frames) < self.num_frames:
lip_frames.append(lip_frames[-1] if lip_frames else torch.zeros(3, self.image_size, self.image_size))
lip_stack = torch.stack(lip_frames) # (T, C, H, W)
# Load mel spectrogram
mel_path = self.cache_dir / video_name / "mel_spectrogram.npy"
if mel_path.exists():
mel = np.load(str(mel_path))
mel_fps = self.sample_rate / 256
mel_per_frame = mel_fps / self.fps
center = int(frame_idx * mel_per_frame)
half_mel = self.num_frames * self.mel_step_size // 2
start = max(0, center - half_mel)
end = min(mel.shape[1], center + half_mel)
mel_seg = mel[:, start:end]
target_w = self.num_frames * self.mel_step_size
if mel_seg.shape[1] < target_w:
mel_seg = np.pad(mel_seg, ((0, 0), (0, target_w - mel_seg.shape[1])))
mel_seg = mel_seg[:, :target_w]
mel_tensor = torch.from_numpy(mel_seg.copy()).float().unsqueeze(0)
else:
mel_tensor = torch.zeros(1, 80, self.num_frames * self.mel_step_size)
# 50% chance positive (in-sync), 50% chance negative (out-of-sync)
is_positive = random.random() > 0.5
if not is_positive:
# Get a random mel from a different frame (out of sync)
neg_idx = random.randint(0, len(self.samples) - 1)
neg_sample = self.samples[neg_idx]
neg_mel_path = self.cache_dir / neg_sample["video_name"] / "mel_spectrogram.npy"
if neg_mel_path.exists():
neg_mel = np.load(str(neg_mel_path))
neg_center = int(neg_sample["frame_idx"] * mel_per_frame)
ns = max(0, neg_center - half_mel)
ne = min(neg_mel.shape[1], neg_center + half_mel)
neg_seg = neg_mel[:, ns:ne]
if neg_seg.shape[1] < target_w:
neg_seg = np.pad(neg_seg, ((0, 0), (0, target_w - neg_seg.shape[1])))
neg_seg = neg_seg[:, :target_w]
mel_tensor = torch.from_numpy(neg_seg.copy()).float().unsqueeze(0)
label = 1.0 if is_positive else 0.0
return {
"lip_frames": lip_stack,
"mel": mel_tensor,
"label": torch.tensor(label),
}

Xet Storage Details

Size:
16.5 kB
·
Xet hash:
03e9d0f0d196d2b767eba7e243f80856e96d0511c4e6980c9de0ac44373c1f9f

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.