Buckets:
| """ | |
| Full LipSync Model Wrapper | |
| ============================ | |
| Combines all components into a single model: | |
| - Frozen VAE (encode/decode) | |
| - Frozen Whisper (audio features) | |
| - Trainable UNet (fusion network) | |
| - Optional frozen SyncNet (sync loss) | |
| This is the top-level module used during training and inference. | |
| """ | |
| from pathlib import Path | |
| from typing import Optional | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from diffusers import AutoencoderKL | |
| from transformers import WhisperModel, WhisperProcessor | |
| from .unet import LipSyncUNet | |
| from .syncnet import SyncNet | |
| class LipSyncModel(nn.Module): | |
| """End-to-end lip sync model. | |
| Training flow: | |
| 1. Encode face image → latent (VAE encoder, frozen) | |
| 2. Create mask (lower half of face) | |
| 3. Apply mask to latent | |
| 4. Extract audio features (Whisper, frozen) | |
| 5. UNet predicts complete latent from masked + reference + audio | |
| 6. Decode latent → image (VAE decoder, frozen) | |
| 7. Compute losses | |
| Inference flow: | |
| Same as training but without loss computation. | |
| """ | |
| def __init__( | |
| self, | |
| unet_config: dict, | |
| vae_model_id: str = "stable-diffusion-v1-5/stable-diffusion-v1-5", | |
| vae_subfolder: str = "vae", | |
| whisper_model_id: str = "openai/whisper-tiny", | |
| syncnet_checkpoint: Optional[str] = None, | |
| device: str = "cuda", | |
| ): | |
| super().__init__() | |
| self.device = device | |
| # === Frozen Components === | |
| # VAE | |
| self.vae = AutoencoderKL.from_pretrained(vae_model_id, subfolder=vae_subfolder) | |
| self.vae.eval() | |
| self.vae.requires_grad_(False) | |
| self.vae_scale_factor = self.vae.config.scaling_factor | |
| # Whisper | |
| self.whisper_processor = WhisperProcessor.from_pretrained(whisper_model_id) | |
| self.whisper = WhisperModel.from_pretrained(whisper_model_id) | |
| self.whisper.eval() | |
| self.whisper.requires_grad_(False) | |
| # SyncNet (optional, for sync loss) | |
| self.syncnet = None | |
| if syncnet_checkpoint and Path(syncnet_checkpoint).exists(): | |
| self.syncnet = SyncNet() | |
| ckpt = torch.load(syncnet_checkpoint, map_location="cpu", weights_only=True) | |
| self.syncnet.load_state_dict(ckpt["model_state_dict"]) | |
| self.syncnet.eval() | |
| self.syncnet.requires_grad_(False) | |
| # === Trainable Component === | |
| self.unet = LipSyncUNet(**unet_config) | |
| def encode_image(self, images: torch.Tensor) -> torch.Tensor: | |
| """Encode images to VAE latent space. | |
| Args: | |
| images: (B, 3, H, W) in [-1, 1] | |
| Returns: | |
| (B, 4, H//8, W//8) latents | |
| """ | |
| with torch.no_grad(): | |
| latent_dist = self.vae.encode(images).latent_dist | |
| latents = latent_dist.sample() * self.vae_scale_factor | |
| return latents | |
| def decode_latent(self, latents: torch.Tensor) -> torch.Tensor: | |
| """Decode VAE latents back to images. | |
| Args: | |
| latents: (B, 4, H//8, W//8) | |
| Returns: | |
| (B, 3, H, W) in [-1, 1] | |
| """ | |
| with torch.no_grad(): | |
| images = self.vae.decode(latents / self.vae_scale_factor).sample | |
| return images | |
| def extract_audio_features( | |
| self, | |
| audio_waveform: torch.Tensor, | |
| sr: int = 16000, | |
| ) -> torch.Tensor: | |
| """Extract Whisper encoder features from audio waveform. | |
| Args: | |
| audio_waveform: (B, N) raw waveform at 16kHz | |
| sr: sample rate | |
| Returns: | |
| (B, T, 384) Whisper encoder features | |
| """ | |
| with torch.no_grad(): | |
| # Process audio through Whisper processor | |
| inputs = self.whisper_processor( | |
| audio_waveform.cpu().numpy(), | |
| sampling_rate=sr, | |
| return_tensors="pt", | |
| ) | |
| input_features = inputs.input_features.to(self.device) | |
| encoder_output = self.whisper.encoder(input_features) | |
| return encoder_output.last_hidden_state | |
| def create_mask( | |
| self, | |
| latents: torch.Tensor, | |
| mask_ratio: float = 0.5, | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| """Create lower-half mask for face latents. | |
| The mask covers the lower half of the face (mouth region). | |
| Args: | |
| latents: (B, 4, H, W) | |
| mask_ratio: fraction of height to mask from bottom | |
| Returns: | |
| masked_latents: (B, 4, H, W) with lower region zeroed | |
| mask: (B, 1, H, W) binary mask (1 = masked/inpaint region) | |
| """ | |
| B, C, H, W = latents.shape | |
| mask = torch.zeros(B, 1, H, W, device=latents.device, dtype=latents.dtype) | |
| mask_start = int(H * (1 - mask_ratio)) | |
| mask[:, :, mask_start:, :] = 1.0 | |
| masked_latents = latents * (1 - mask) | |
| return masked_latents, mask | |
| def forward( | |
| self, | |
| target_latents: torch.Tensor, | |
| reference_latents: torch.Tensor, | |
| audio_features: torch.Tensor, | |
| mask_ratio: float = 0.5, | |
| ) -> dict: | |
| """ | |
| Training forward pass. | |
| Args: | |
| target_latents: (B, 4, H, W) ground truth face latents | |
| reference_latents: (B, 4, H, W) reference face latents (identity) | |
| audio_features: (B, T, D) pre-extracted Whisper features | |
| mask_ratio: fraction to mask | |
| Returns: | |
| dict with predicted latents and intermediate results | |
| """ | |
| # Create mask | |
| masked_latents, mask = self.create_mask(target_latents, mask_ratio) | |
| # UNet prediction | |
| predicted_latents = self.unet( | |
| masked_latent=masked_latents, | |
| reference_latent=reference_latents, | |
| mask=mask, | |
| audio_features=audio_features, | |
| ) | |
| return { | |
| "predicted_latents": predicted_latents, | |
| "target_latents": target_latents, | |
| "masked_latents": masked_latents, | |
| "mask": mask, | |
| "reference_latents": reference_latents, | |
| } | |
| def generate( | |
| self, | |
| reference_image: torch.Tensor, | |
| audio_features: torch.Tensor, | |
| mask_ratio: float = 0.5, | |
| ) -> torch.Tensor: | |
| """ | |
| Inference: generate lip-synced face from reference + audio. | |
| Args: | |
| reference_image: (B, 3, H, W) in [-1, 1] | |
| audio_features: (B, T, D) Whisper features | |
| mask_ratio: fraction to mask | |
| Returns: | |
| (B, 3, H, W) generated image in [-1, 1] | |
| """ | |
| # Encode reference | |
| ref_latents = self.encode_image(reference_image) | |
| # Create mask | |
| masked_latents, mask = self.create_mask(ref_latents, mask_ratio) | |
| # Predict | |
| predicted_latents = self.unet( | |
| masked_latent=masked_latents, | |
| reference_latent=ref_latents, | |
| mask=mask, | |
| audio_features=audio_features, | |
| ) | |
| # Decode | |
| generated = self.decode_latent(predicted_latents) | |
| return generated.clamp(-1, 1) | |
| def get_trainable_parameters(self): | |
| """Return only trainable parameters (UNet).""" | |
| return self.unet.parameters() | |
| def get_param_summary(self) -> dict: | |
| """Get parameter count summary.""" | |
| total = sum(p.numel() for p in self.parameters()) | |
| trainable = sum(p.numel() for p in self.parameters() if p.requires_grad) | |
| frozen = total - trainable | |
| return { | |
| "total_M": total / 1e6, | |
| "trainable_M": trainable / 1e6, | |
| "frozen_M": frozen / 1e6, | |
| "unet": self.unet.get_param_count(), | |
| } | |
Xet Storage Details
- Size:
- 8.01 kB
- Xet hash:
- c8cde8c71f62c45d658deccc0524a5e865aed473d6be6937b2940ccee224743d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.