Buckets:
| """ | |
| SyncNet - Lip Sync Discriminator | |
| ================================== | |
| A contrastive network that learns whether audio and lip movements are in sync. | |
| Used both as: | |
| 1. A standalone model (pre-trained on the dataset) | |
| 2. A frozen loss function during main model training | |
| Architecture: | |
| - Visual encoder: processes lip region crops | |
| - Audio encoder: processes mel spectrograms | |
| - Contrastive loss: in-sync pairs should have high cosine similarity | |
| Based on "Out of time: automated lip sync in the wild" (Chung & Zisserman, 2016) | |
| but modernized with better architecture choices. | |
| """ | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class ConvBlock(nn.Module): | |
| """Conv -> BatchNorm -> ReLU block.""" | |
| def __init__( | |
| self, | |
| in_ch: int, | |
| out_ch: int, | |
| kernel_size: int = 3, | |
| stride: int = 1, | |
| padding: int = 1, | |
| ): | |
| super().__init__() | |
| self.conv = nn.Conv2d(in_ch, out_ch, kernel_size, stride, padding) | |
| self.bn = nn.BatchNorm2d(out_ch) | |
| self.relu = nn.ReLU(inplace=True) | |
| def forward(self, x): | |
| return self.relu(self.bn(self.conv(x))) | |
| class VisualEncoder(nn.Module): | |
| """Encodes a sequence of lip crops into a fixed embedding. | |
| Input: (B, T, C, H, W) - T frames of lip region crops | |
| Output: (B, embedding_dim) - visual embedding | |
| """ | |
| def __init__( | |
| self, | |
| in_channels: int = 3, | |
| image_size: int = 96, | |
| embedding_dim: int = 512, | |
| num_frames: int = 5, | |
| ): | |
| super().__init__() | |
| self.num_frames = num_frames | |
| # Per-frame 2D encoder | |
| self.frame_encoder = nn.Sequential( | |
| ConvBlock(in_channels, 64, 7, 1, 3), | |
| ConvBlock(64, 64, 3, 1, 1), | |
| nn.MaxPool2d(2, 2), # 48x48 | |
| ConvBlock(64, 128, 3, 1, 1), | |
| ConvBlock(128, 128, 3, 1, 1), | |
| nn.MaxPool2d(2, 2), # 24x24 | |
| ConvBlock(128, 256, 3, 1, 1), | |
| ConvBlock(256, 256, 3, 1, 1), | |
| nn.MaxPool2d(2, 2), # 12x12 | |
| ConvBlock(256, 512, 3, 1, 1), | |
| ConvBlock(512, 512, 3, 1, 1), | |
| nn.AdaptiveAvgPool2d((4, 4)), # 4x4 | |
| ) | |
| # Temporal aggregation (across frames) | |
| self.temporal_conv = nn.Sequential( | |
| nn.Conv1d(512 * 16, 512, kernel_size=num_frames, padding=0), | |
| nn.BatchNorm1d(512), | |
| nn.ReLU(inplace=True), | |
| ) | |
| self.fc = nn.Linear(512, embedding_dim) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Args: | |
| x: (B, T, C, H, W) lip frames | |
| Returns: | |
| (B, embedding_dim) | |
| """ | |
| B, T, C, H, W = x.shape | |
| # Encode each frame | |
| x = x.reshape(B * T, C, H, W) | |
| x = self.frame_encoder(x) # (B*T, 512, 4, 4) | |
| x = x.reshape(B, T, -1) # (B, T, 512*16) | |
| # Temporal aggregation | |
| x = x.transpose(1, 2) # (B, 512*16, T) | |
| x = self.temporal_conv(x) # (B, 512, 1) | |
| x = x.squeeze(-1) # (B, 512) | |
| x = self.fc(x) # (B, embedding_dim) | |
| return F.normalize(x, dim=-1) | |
| class AudioEncoder(nn.Module): | |
| """Encodes mel spectrogram into a fixed embedding. | |
| Input: (B, 1, mel_bins, mel_steps) - mel spectrogram segment | |
| Output: (B, embedding_dim) - audio embedding | |
| """ | |
| def __init__( | |
| self, | |
| mel_bins: int = 80, | |
| mel_steps: int = 80, # num_audio_frames * mel_step_size | |
| embedding_dim: int = 512, | |
| ): | |
| super().__init__() | |
| self.encoder = nn.Sequential( | |
| ConvBlock(1, 64, 3, 1, 1), | |
| ConvBlock(64, 64, 3, 1, 1), | |
| nn.MaxPool2d((1, 2)), # -> (80, 40) | |
| ConvBlock(64, 128, 3, 1, 1), | |
| ConvBlock(128, 128, 3, 1, 1), | |
| nn.MaxPool2d((2, 2)), # -> (40, 20) | |
| ConvBlock(128, 256, 3, 1, 1), | |
| ConvBlock(256, 256, 3, 1, 1), | |
| nn.MaxPool2d((2, 2)), # -> (20, 10) | |
| ConvBlock(256, 512, 3, 1, 1), | |
| ConvBlock(512, 512, 3, 1, 1), | |
| nn.AdaptiveAvgPool2d((1, 1)), | |
| ) | |
| self.fc = nn.Linear(512, embedding_dim) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Args: | |
| x: (B, 1, mel_bins, mel_steps) mel spectrogram | |
| Returns: | |
| (B, embedding_dim) | |
| """ | |
| x = self.encoder(x) | |
| x = x.squeeze(-1).squeeze(-1) # (B, 512) | |
| x = self.fc(x) | |
| return F.normalize(x, dim=-1) | |
| class SyncNet(nn.Module): | |
| """SyncNet: learns audio-visual synchronization. | |
| Trained with contrastive loss: | |
| - Positive pairs: video frames + corresponding audio (in sync) | |
| - Negative pairs: video frames + random audio (out of sync) | |
| During main model training, this is frozen and used to compute | |
| a sync loss that encourages generated lips to match the audio. | |
| """ | |
| def __init__( | |
| self, | |
| image_channels: int = 3, | |
| image_size: int = 96, | |
| embedding_dim: int = 512, | |
| num_frames: int = 5, | |
| mel_bins: int = 80, | |
| mel_steps: int = 80, | |
| ): | |
| super().__init__() | |
| self.visual_encoder = VisualEncoder( | |
| in_channels=image_channels, | |
| image_size=image_size, | |
| embedding_dim=embedding_dim, | |
| num_frames=num_frames, | |
| ) | |
| self.audio_encoder = AudioEncoder( | |
| mel_bins=mel_bins, | |
| mel_steps=mel_steps, | |
| embedding_dim=embedding_dim, | |
| ) | |
| self.logit_scale = nn.Parameter(torch.ones([]) * 2.6592) # ln(1/0.07) | |
| def forward( | |
| self, | |
| video: torch.Tensor, | |
| audio: torch.Tensor, | |
| ) -> dict: | |
| """ | |
| Args: | |
| video: (B, T, C, H, W) lip region frames | |
| audio: (B, 1, mel_bins, mel_steps) mel spectrogram | |
| Returns: | |
| dict with embeddings and similarity logits | |
| """ | |
| video_emb = self.visual_encoder(video) # (B, D) | |
| audio_emb = self.audio_encoder(audio) # (B, D) | |
| # Cosine similarity with learnable temperature | |
| logit_scale = self.logit_scale.exp().clamp(max=100) | |
| logits = logit_scale * (video_emb * audio_emb).sum(dim=-1) | |
| return { | |
| "video_embedding": video_emb, | |
| "audio_embedding": audio_emb, | |
| "logits": logits, | |
| "cosine_similarity": (video_emb * audio_emb).sum(dim=-1), | |
| } | |
| def sync_loss( | |
| self, | |
| video: torch.Tensor, | |
| audio: torch.Tensor, | |
| ) -> torch.Tensor: | |
| """Compute sync loss for generated frames. | |
| Used during main model training (SyncNet is frozen). | |
| Higher sync loss means lips are out of sync with audio. | |
| """ | |
| output = self.forward(video, audio) | |
| # We want high cosine similarity for generated frames | |
| # So loss = 1 - cosine_similarity (minimizing pushes similarity up) | |
| return 1.0 - output["cosine_similarity"].mean() | |
| def contrastive_loss( | |
| video_emb: torch.Tensor, | |
| audio_emb: torch.Tensor, | |
| logit_scale: torch.Tensor, | |
| ) -> torch.Tensor: | |
| """InfoNCE contrastive loss for SyncNet training. | |
| Each sample in the batch serves as negative for all other samples. | |
| """ | |
| logit_scale = logit_scale.exp().clamp(max=100) | |
| # Compute similarity matrix | |
| logits_va = logit_scale * video_emb @ audio_emb.t() # (B, B) | |
| logits_av = logits_va.t() | |
| batch_size = video_emb.shape[0] | |
| labels = torch.arange(batch_size, device=video_emb.device) | |
| loss_va = F.cross_entropy(logits_va, labels) | |
| loss_av = F.cross_entropy(logits_av, labels) | |
| return (loss_va + loss_av) / 2 | |
Xet Storage Details
- Size:
- 8.02 kB
- Xet hash:
- b7031d0dd2141a8224553e4a8a10eb6cbf8df773b475cbc7f311bde1dc0b27ab
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.