minhluanquach's picture
download
raw
8.02 kB
"""
SyncNet - Lip Sync Discriminator
==================================
A contrastive network that learns whether audio and lip movements are in sync.
Used both as:
1. A standalone model (pre-trained on the dataset)
2. A frozen loss function during main model training
Architecture:
- Visual encoder: processes lip region crops
- Audio encoder: processes mel spectrograms
- Contrastive loss: in-sync pairs should have high cosine similarity
Based on "Out of time: automated lip sync in the wild" (Chung & Zisserman, 2016)
but modernized with better architecture choices.
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
class ConvBlock(nn.Module):
"""Conv -> BatchNorm -> ReLU block."""
def __init__(
self,
in_ch: int,
out_ch: int,
kernel_size: int = 3,
stride: int = 1,
padding: int = 1,
):
super().__init__()
self.conv = nn.Conv2d(in_ch, out_ch, kernel_size, stride, padding)
self.bn = nn.BatchNorm2d(out_ch)
self.relu = nn.ReLU(inplace=True)
def forward(self, x):
return self.relu(self.bn(self.conv(x)))
class VisualEncoder(nn.Module):
"""Encodes a sequence of lip crops into a fixed embedding.
Input: (B, T, C, H, W) - T frames of lip region crops
Output: (B, embedding_dim) - visual embedding
"""
def __init__(
self,
in_channels: int = 3,
image_size: int = 96,
embedding_dim: int = 512,
num_frames: int = 5,
):
super().__init__()
self.num_frames = num_frames
# Per-frame 2D encoder
self.frame_encoder = nn.Sequential(
ConvBlock(in_channels, 64, 7, 1, 3),
ConvBlock(64, 64, 3, 1, 1),
nn.MaxPool2d(2, 2), # 48x48
ConvBlock(64, 128, 3, 1, 1),
ConvBlock(128, 128, 3, 1, 1),
nn.MaxPool2d(2, 2), # 24x24
ConvBlock(128, 256, 3, 1, 1),
ConvBlock(256, 256, 3, 1, 1),
nn.MaxPool2d(2, 2), # 12x12
ConvBlock(256, 512, 3, 1, 1),
ConvBlock(512, 512, 3, 1, 1),
nn.AdaptiveAvgPool2d((4, 4)), # 4x4
)
# Temporal aggregation (across frames)
self.temporal_conv = nn.Sequential(
nn.Conv1d(512 * 16, 512, kernel_size=num_frames, padding=0),
nn.BatchNorm1d(512),
nn.ReLU(inplace=True),
)
self.fc = nn.Linear(512, embedding_dim)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
Args:
x: (B, T, C, H, W) lip frames
Returns:
(B, embedding_dim)
"""
B, T, C, H, W = x.shape
# Encode each frame
x = x.reshape(B * T, C, H, W)
x = self.frame_encoder(x) # (B*T, 512, 4, 4)
x = x.reshape(B, T, -1) # (B, T, 512*16)
# Temporal aggregation
x = x.transpose(1, 2) # (B, 512*16, T)
x = self.temporal_conv(x) # (B, 512, 1)
x = x.squeeze(-1) # (B, 512)
x = self.fc(x) # (B, embedding_dim)
return F.normalize(x, dim=-1)
class AudioEncoder(nn.Module):
"""Encodes mel spectrogram into a fixed embedding.
Input: (B, 1, mel_bins, mel_steps) - mel spectrogram segment
Output: (B, embedding_dim) - audio embedding
"""
def __init__(
self,
mel_bins: int = 80,
mel_steps: int = 80, # num_audio_frames * mel_step_size
embedding_dim: int = 512,
):
super().__init__()
self.encoder = nn.Sequential(
ConvBlock(1, 64, 3, 1, 1),
ConvBlock(64, 64, 3, 1, 1),
nn.MaxPool2d((1, 2)), # -> (80, 40)
ConvBlock(64, 128, 3, 1, 1),
ConvBlock(128, 128, 3, 1, 1),
nn.MaxPool2d((2, 2)), # -> (40, 20)
ConvBlock(128, 256, 3, 1, 1),
ConvBlock(256, 256, 3, 1, 1),
nn.MaxPool2d((2, 2)), # -> (20, 10)
ConvBlock(256, 512, 3, 1, 1),
ConvBlock(512, 512, 3, 1, 1),
nn.AdaptiveAvgPool2d((1, 1)),
)
self.fc = nn.Linear(512, embedding_dim)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
Args:
x: (B, 1, mel_bins, mel_steps) mel spectrogram
Returns:
(B, embedding_dim)
"""
x = self.encoder(x)
x = x.squeeze(-1).squeeze(-1) # (B, 512)
x = self.fc(x)
return F.normalize(x, dim=-1)
class SyncNet(nn.Module):
"""SyncNet: learns audio-visual synchronization.
Trained with contrastive loss:
- Positive pairs: video frames + corresponding audio (in sync)
- Negative pairs: video frames + random audio (out of sync)
During main model training, this is frozen and used to compute
a sync loss that encourages generated lips to match the audio.
"""
def __init__(
self,
image_channels: int = 3,
image_size: int = 96,
embedding_dim: int = 512,
num_frames: int = 5,
mel_bins: int = 80,
mel_steps: int = 80,
):
super().__init__()
self.visual_encoder = VisualEncoder(
in_channels=image_channels,
image_size=image_size,
embedding_dim=embedding_dim,
num_frames=num_frames,
)
self.audio_encoder = AudioEncoder(
mel_bins=mel_bins,
mel_steps=mel_steps,
embedding_dim=embedding_dim,
)
self.logit_scale = nn.Parameter(torch.ones([]) * 2.6592) # ln(1/0.07)
def forward(
self,
video: torch.Tensor,
audio: torch.Tensor,
) -> dict:
"""
Args:
video: (B, T, C, H, W) lip region frames
audio: (B, 1, mel_bins, mel_steps) mel spectrogram
Returns:
dict with embeddings and similarity logits
"""
video_emb = self.visual_encoder(video) # (B, D)
audio_emb = self.audio_encoder(audio) # (B, D)
# Cosine similarity with learnable temperature
logit_scale = self.logit_scale.exp().clamp(max=100)
logits = logit_scale * (video_emb * audio_emb).sum(dim=-1)
return {
"video_embedding": video_emb,
"audio_embedding": audio_emb,
"logits": logits,
"cosine_similarity": (video_emb * audio_emb).sum(dim=-1),
}
def sync_loss(
self,
video: torch.Tensor,
audio: torch.Tensor,
) -> torch.Tensor:
"""Compute sync loss for generated frames.
Used during main model training (SyncNet is frozen).
Higher sync loss means lips are out of sync with audio.
"""
output = self.forward(video, audio)
# We want high cosine similarity for generated frames
# So loss = 1 - cosine_similarity (minimizing pushes similarity up)
return 1.0 - output["cosine_similarity"].mean()
@staticmethod
def contrastive_loss(
video_emb: torch.Tensor,
audio_emb: torch.Tensor,
logit_scale: torch.Tensor,
) -> torch.Tensor:
"""InfoNCE contrastive loss for SyncNet training.
Each sample in the batch serves as negative for all other samples.
"""
logit_scale = logit_scale.exp().clamp(max=100)
# Compute similarity matrix
logits_va = logit_scale * video_emb @ audio_emb.t() # (B, B)
logits_av = logits_va.t()
batch_size = video_emb.shape[0]
labels = torch.arange(batch_size, device=video_emb.device)
loss_va = F.cross_entropy(logits_va, labels)
loss_av = F.cross_entropy(logits_av, labels)
return (loss_va + loss_av) / 2

Xet Storage Details

Size:
8.02 kB
·
Xet hash:
b7031d0dd2141a8224553e4a8a10eb6cbf8df773b475cbc7f311bde1dc0b27ab

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.