minhluanquach's picture
download
raw
8.01 kB
"""
Full LipSync Model Wrapper
============================
Combines all components into a single model:
- Frozen VAE (encode/decode)
- Frozen Whisper (audio features)
- Trainable UNet (fusion network)
- Optional frozen SyncNet (sync loss)
This is the top-level module used during training and inference.
"""
from pathlib import Path
from typing import Optional
import torch
import torch.nn as nn
import torch.nn.functional as F
from diffusers import AutoencoderKL
from transformers import WhisperModel, WhisperProcessor
from .unet import LipSyncUNet
from .syncnet import SyncNet
class LipSyncModel(nn.Module):
"""End-to-end lip sync model.
Training flow:
1. Encode face image → latent (VAE encoder, frozen)
2. Create mask (lower half of face)
3. Apply mask to latent
4. Extract audio features (Whisper, frozen)
5. UNet predicts complete latent from masked + reference + audio
6. Decode latent → image (VAE decoder, frozen)
7. Compute losses
Inference flow:
Same as training but without loss computation.
"""
def __init__(
self,
unet_config: dict,
vae_model_id: str = "stable-diffusion-v1-5/stable-diffusion-v1-5",
vae_subfolder: str = "vae",
whisper_model_id: str = "openai/whisper-tiny",
syncnet_checkpoint: Optional[str] = None,
device: str = "cuda",
):
super().__init__()
self.device = device
# === Frozen Components ===
# VAE
self.vae = AutoencoderKL.from_pretrained(vae_model_id, subfolder=vae_subfolder)
self.vae.eval()
self.vae.requires_grad_(False)
self.vae_scale_factor = self.vae.config.scaling_factor
# Whisper
self.whisper_processor = WhisperProcessor.from_pretrained(whisper_model_id)
self.whisper = WhisperModel.from_pretrained(whisper_model_id)
self.whisper.eval()
self.whisper.requires_grad_(False)
# SyncNet (optional, for sync loss)
self.syncnet = None
if syncnet_checkpoint and Path(syncnet_checkpoint).exists():
self.syncnet = SyncNet()
ckpt = torch.load(syncnet_checkpoint, map_location="cpu", weights_only=True)
self.syncnet.load_state_dict(ckpt["model_state_dict"])
self.syncnet.eval()
self.syncnet.requires_grad_(False)
# === Trainable Component ===
self.unet = LipSyncUNet(**unet_config)
def encode_image(self, images: torch.Tensor) -> torch.Tensor:
"""Encode images to VAE latent space.
Args:
images: (B, 3, H, W) in [-1, 1]
Returns:
(B, 4, H//8, W//8) latents
"""
with torch.no_grad():
latent_dist = self.vae.encode(images).latent_dist
latents = latent_dist.sample() * self.vae_scale_factor
return latents
def decode_latent(self, latents: torch.Tensor) -> torch.Tensor:
"""Decode VAE latents back to images.
Args:
latents: (B, 4, H//8, W//8)
Returns:
(B, 3, H, W) in [-1, 1]
"""
with torch.no_grad():
images = self.vae.decode(latents / self.vae_scale_factor).sample
return images
def extract_audio_features(
self,
audio_waveform: torch.Tensor,
sr: int = 16000,
) -> torch.Tensor:
"""Extract Whisper encoder features from audio waveform.
Args:
audio_waveform: (B, N) raw waveform at 16kHz
sr: sample rate
Returns:
(B, T, 384) Whisper encoder features
"""
with torch.no_grad():
# Process audio through Whisper processor
inputs = self.whisper_processor(
audio_waveform.cpu().numpy(),
sampling_rate=sr,
return_tensors="pt",
)
input_features = inputs.input_features.to(self.device)
encoder_output = self.whisper.encoder(input_features)
return encoder_output.last_hidden_state
def create_mask(
self,
latents: torch.Tensor,
mask_ratio: float = 0.5,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Create lower-half mask for face latents.
The mask covers the lower half of the face (mouth region).
Args:
latents: (B, 4, H, W)
mask_ratio: fraction of height to mask from bottom
Returns:
masked_latents: (B, 4, H, W) with lower region zeroed
mask: (B, 1, H, W) binary mask (1 = masked/inpaint region)
"""
B, C, H, W = latents.shape
mask = torch.zeros(B, 1, H, W, device=latents.device, dtype=latents.dtype)
mask_start = int(H * (1 - mask_ratio))
mask[:, :, mask_start:, :] = 1.0
masked_latents = latents * (1 - mask)
return masked_latents, mask
def forward(
self,
target_latents: torch.Tensor,
reference_latents: torch.Tensor,
audio_features: torch.Tensor,
mask_ratio: float = 0.5,
) -> dict:
"""
Training forward pass.
Args:
target_latents: (B, 4, H, W) ground truth face latents
reference_latents: (B, 4, H, W) reference face latents (identity)
audio_features: (B, T, D) pre-extracted Whisper features
mask_ratio: fraction to mask
Returns:
dict with predicted latents and intermediate results
"""
# Create mask
masked_latents, mask = self.create_mask(target_latents, mask_ratio)
# UNet prediction
predicted_latents = self.unet(
masked_latent=masked_latents,
reference_latent=reference_latents,
mask=mask,
audio_features=audio_features,
)
return {
"predicted_latents": predicted_latents,
"target_latents": target_latents,
"masked_latents": masked_latents,
"mask": mask,
"reference_latents": reference_latents,
}
@torch.no_grad()
def generate(
self,
reference_image: torch.Tensor,
audio_features: torch.Tensor,
mask_ratio: float = 0.5,
) -> torch.Tensor:
"""
Inference: generate lip-synced face from reference + audio.
Args:
reference_image: (B, 3, H, W) in [-1, 1]
audio_features: (B, T, D) Whisper features
mask_ratio: fraction to mask
Returns:
(B, 3, H, W) generated image in [-1, 1]
"""
# Encode reference
ref_latents = self.encode_image(reference_image)
# Create mask
masked_latents, mask = self.create_mask(ref_latents, mask_ratio)
# Predict
predicted_latents = self.unet(
masked_latent=masked_latents,
reference_latent=ref_latents,
mask=mask,
audio_features=audio_features,
)
# Decode
generated = self.decode_latent(predicted_latents)
return generated.clamp(-1, 1)
def get_trainable_parameters(self):
"""Return only trainable parameters (UNet)."""
return self.unet.parameters()
def get_param_summary(self) -> dict:
"""Get parameter count summary."""
total = sum(p.numel() for p in self.parameters())
trainable = sum(p.numel() for p in self.parameters() if p.requires_grad)
frozen = total - trainable
return {
"total_M": total / 1e6,
"trainable_M": trainable / 1e6,
"frozen_M": frozen / 1e6,
"unet": self.unet.get_param_count(),
}

Xet Storage Details

Size:
8.01 kB
·
Xet hash:
c8cde8c71f62c45d658deccc0524a5e865aed473d6be6937b2940ccee224743d

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.