Buckets:
| """ | |
| Inference Pipeline | |
| ==================== | |
| Takes a video + audio and produces a lip-synced output video. | |
| Pipeline: | |
| 1. Extract faces from video frames | |
| 2. Extract Whisper features from audio | |
| 3. For each frame: encode → mask → fuse with audio → decode | |
| 4. Paste generated face back onto original frame | |
| 5. Combine frames into output video with original audio | |
| Usage: | |
| python -m src.inference.predict \ | |
| --video input.mp4 \ | |
| --audio speech.wav \ | |
| --checkpoint checkpoints/best_model.pt \ | |
| --output result.mp4 | |
| """ | |
| import argparse | |
| import os | |
| import tempfile | |
| from pathlib import Path | |
| import cv2 | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from PIL import Image | |
| from torchvision import transforms | |
| from tqdm import tqdm | |
| from src.models.unet import LipSyncUNet | |
| class LipSyncInference: | |
| """Complete inference pipeline for lip synchronization.""" | |
| def __init__( | |
| self, | |
| checkpoint_path: str, | |
| vae_model_id: str = "stable-diffusion-v1-5/stable-diffusion-v1-5", | |
| whisper_model_id: str = "openai/whisper-tiny", | |
| device: str = "cuda", | |
| image_size: int = 256, | |
| use_ema: bool = True, | |
| ): | |
| self.device = torch.device(device if torch.cuda.is_available() else "cpu") | |
| self.image_size = image_size | |
| print(f"Using device: {self.device}") | |
| # Load UNet | |
| print("Loading UNet...") | |
| ckpt = torch.load(checkpoint_path, map_location=self.device, weights_only=False) | |
| unet_config = ckpt["config"]["model"]["unet"] | |
| self.unet = LipSyncUNet(**unet_config).to(self.device) | |
| # Load EMA weights if available | |
| if use_ema and "ema_state_dict" in ckpt: | |
| print("Using EMA weights") | |
| ema_shadow = ckpt["ema_state_dict"]["shadow"] | |
| state_dict = self.unet.state_dict() | |
| for name in ema_shadow: | |
| if name in state_dict: | |
| state_dict[name] = ema_shadow[name] | |
| self.unet.load_state_dict(state_dict) | |
| else: | |
| self.unet.load_state_dict(ckpt["model_state_dict"]) | |
| self.unet.eval() | |
| self.unet.requires_grad_(False) | |
| # Load VAE | |
| print("Loading VAE...") | |
| from diffusers import AutoencoderKL | |
| self.vae = AutoencoderKL.from_pretrained( | |
| vae_model_id, subfolder="vae" | |
| ).to(self.device) | |
| self.vae.eval() | |
| self.vae.requires_grad_(False) | |
| self.vae_scale = self.vae.config.scaling_factor | |
| # Load Whisper | |
| print("Loading Whisper...") | |
| from transformers import WhisperModel, WhisperProcessor | |
| self.whisper_processor = WhisperProcessor.from_pretrained(whisper_model_id) | |
| self.whisper = WhisperModel.from_pretrained(whisper_model_id).to(self.device) | |
| self.whisper.eval() | |
| self.whisper.requires_grad_(False) | |
| # Face detector | |
| print("Loading face detector...") | |
| import mediapipe as mp | |
| model_path = Path(__file__).resolve().parent.parent.parent / "models" / "face_landmarker.task" | |
| base_options = mp.tasks.BaseOptions(model_asset_path=str(model_path)) | |
| options = mp.tasks.vision.FaceLandmarkerOptions( | |
| base_options=base_options, | |
| running_mode=mp.tasks.vision.RunningMode.IMAGE, | |
| num_faces=1, | |
| min_face_detection_confidence=0.5, | |
| min_face_presence_confidence=0.5, | |
| ) | |
| self.face_landmarker = mp.tasks.vision.FaceLandmarker.create_from_options(options) | |
| # Image transform | |
| self.transform = transforms.Compose([ | |
| transforms.Resize(image_size), | |
| transforms.CenterCrop(image_size), | |
| transforms.ToTensor(), | |
| transforms.Normalize([0.5], [0.5]), | |
| ]) | |
| print("Inference pipeline ready!") | |
| def detect_face_bbox(self, frame: np.ndarray) -> list[int]: | |
| """Detect face bounding box in a frame.""" | |
| import mediapipe as mp | |
| rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) | |
| mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb) | |
| result = self.face_landmarker.detect(mp_image) | |
| if not result.face_landmarks: | |
| return None | |
| landmarks = result.face_landmarks[0] | |
| h, w = frame.shape[:2] | |
| points = np.array([(lm.x * w, lm.y * h) for lm in landmarks]) | |
| x_min, y_min = points.min(axis=0) | |
| x_max, y_max = points.max(axis=0) | |
| face_w = x_max - x_min | |
| face_h = y_max - y_min | |
| pad = max(face_w, face_h) * 0.2 | |
| cx = (x_min + x_max) / 2 | |
| cy = (y_min + y_max) / 2 | |
| size = max(face_w, face_h) + 2 * pad | |
| half = size / 2 | |
| x1 = int(max(0, cx - half)) | |
| y1 = int(max(0, cy - half)) | |
| x2 = int(min(w, cx + half)) | |
| y2 = int(min(h, cy + half)) | |
| return [x1, y1, x2, y2] | |
| def extract_whisper_features(self, audio_path: str) -> torch.Tensor: | |
| """Extract Whisper encoder features from audio file.""" | |
| import librosa | |
| audio, sr = librosa.load(audio_path, sr=16000, mono=True) | |
| # Process in 30s chunks | |
| chunk_size = 30 * sr | |
| all_features = [] | |
| for start in range(0, len(audio), chunk_size): | |
| chunk = audio[start:start + chunk_size] | |
| if len(chunk) < sr: | |
| continue | |
| inputs = self.whisper_processor(chunk, sampling_rate=sr, return_tensors="pt") | |
| input_features = inputs.input_features.to(self.device) | |
| with torch.no_grad(): | |
| features = self.whisper.encoder(input_features).last_hidden_state | |
| all_features.append(features.cpu()) | |
| if not all_features: | |
| return torch.zeros(1, 1, 384).to(self.device) | |
| return torch.cat(all_features, dim=1).to(self.device) | |
| def process_frame( | |
| self, | |
| face_crop: np.ndarray, | |
| audio_features: torch.Tensor, | |
| reference_latent: torch.Tensor, | |
| ) -> np.ndarray: | |
| """Process a single face crop with audio features.""" | |
| # Prep image | |
| img = Image.fromarray(cv2.cvtColor(face_crop, cv2.COLOR_BGR2RGB)) | |
| img_tensor = self.transform(img).unsqueeze(0).to(self.device) | |
| # Encode to latent | |
| latent = self.vae.encode(img_tensor).latent_dist.sample() * self.vae_scale | |
| # Create mask (lower half) | |
| B, C, H, W = latent.shape | |
| mask = torch.zeros(B, 1, H, W, device=self.device) | |
| mask[:, :, H // 2:, :] = 1.0 | |
| masked = latent * (1 - mask) | |
| # UNet prediction | |
| pred_latent = self.unet( | |
| masked_latent=masked, | |
| reference_latent=reference_latent, | |
| mask=mask, | |
| audio_features=audio_features, | |
| ) | |
| # Decode | |
| decoded = self.vae.decode(pred_latent / self.vae_scale).sample | |
| decoded = decoded.clamp(-1, 1) | |
| # Blend: keep original upper half, use predicted lower half | |
| # This eliminates the visible mask seam from latent-space masking | |
| pixel_h = img_tensor.shape[2] | |
| blend_start = pixel_h // 2 | |
| feather = max(4, pixel_h // 32) | |
| blended = img_tensor.clone() | |
| blended[:, :, blend_start:, :] = decoded[:, :, blend_start:, :] | |
| for i in range(feather): | |
| alpha = i / feather | |
| row = blend_start - feather + i | |
| if 0 <= row < pixel_h: | |
| blended[:, :, row, :] = ( | |
| (1 - alpha) * img_tensor[:, :, row, :] | |
| + alpha * decoded[:, :, row, :] | |
| ) | |
| # Convert to numpy image | |
| img_out = ((blended[0].permute(1, 2, 0).cpu().numpy() + 1) * 127.5).astype(np.uint8) | |
| img_out = cv2.cvtColor(img_out, cv2.COLOR_RGB2BGR) | |
| img_out = cv2.resize(img_out, (face_crop.shape[1], face_crop.shape[0])) | |
| return img_out | |
| def paste_face( | |
| self, | |
| frame: np.ndarray, | |
| face: np.ndarray, | |
| bbox: list[int], | |
| ) -> np.ndarray: | |
| """Paste generated face back onto original frame with blending.""" | |
| x1, y1, x2, y2 = bbox | |
| h, w = y2 - y1, x2 - x1 | |
| face_resized = cv2.resize(face, (w, h)) | |
| result = frame.copy() | |
| # Create a smooth blending mask (feathered edges) | |
| blend_mask = np.ones((h, w), dtype=np.float32) | |
| feather = min(h, w) // 10 | |
| if feather > 0: | |
| for i in range(feather): | |
| alpha = i / feather | |
| blend_mask[i, :] *= alpha | |
| blend_mask[h - 1 - i, :] *= alpha | |
| blend_mask[:, i] *= alpha | |
| blend_mask[:, w - 1 - i] *= alpha | |
| blend_mask = blend_mask[:, :, np.newaxis] | |
| # Only blend the lower half (where lips are modified) | |
| mid_y = h // 2 | |
| blend_mask[:mid_y] = 0 # Don't change upper half | |
| result[y1:y2, x1:x2] = ( | |
| result[y1:y2, x1:x2] * (1 - blend_mask) | |
| + face_resized * blend_mask | |
| ).astype(np.uint8) | |
| return result | |
| def run( | |
| self, | |
| video_path: str, | |
| audio_path: str, | |
| output_path: str, | |
| fps: int = 25, | |
| batch_size: int = 4, | |
| ): | |
| """Run full inference pipeline.""" | |
| print(f"\nProcessing video: {video_path}") | |
| print(f"Audio: {audio_path}") | |
| # Extract audio features | |
| print("Extracting audio features...") | |
| all_audio_features = self.extract_whisper_features(audio_path) | |
| # Open video | |
| cap = cv2.VideoCapture(video_path) | |
| original_fps = cap.get(cv2.CAP_PROP_FPS) or fps | |
| total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) | |
| frame_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) | |
| frame_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) | |
| # Output writer | |
| output_path_temp = output_path + ".temp.mp4" | |
| fourcc = cv2.VideoWriter_fourcc(*"mp4v") | |
| writer = cv2.VideoWriter(output_path_temp, fourcc, original_fps, (frame_w, frame_h)) | |
| # Get reference frame (first frame with detectable face) | |
| ref_latent = None | |
| cap_ref = cv2.VideoCapture(video_path) | |
| for _ in range(min(30, total_frames)): | |
| ret, frame = cap_ref.read() | |
| if not ret: | |
| break | |
| bbox = self.detect_face_bbox(frame) | |
| if bbox: | |
| face_crop = frame[bbox[1]:bbox[3], bbox[0]:bbox[2]] | |
| face_crop = cv2.resize(face_crop, (self.image_size, self.image_size)) | |
| img = Image.fromarray(cv2.cvtColor(face_crop, cv2.COLOR_BGR2RGB)) | |
| img_t = self.transform(img).unsqueeze(0).to(self.device) | |
| ref_latent = self.vae.encode(img_t).latent_dist.sample() * self.vae_scale | |
| break | |
| cap_ref.release() | |
| if ref_latent is None: | |
| print("ERROR: Could not detect face in video for reference") | |
| cap.release() | |
| return | |
| # Process frames | |
| whisper_fps = 1500 / 30 # 50 features/sec | |
| # Half-window must match training: num_audio_frames * 4 / 2 = 10 | |
| audio_context_window = 10 | |
| print(f"Processing {total_frames} frames...") | |
| frame_idx = 0 | |
| for _ in tqdm(range(total_frames), desc="Generating"): | |
| ret, frame = cap.read() | |
| if not ret: | |
| break | |
| bbox = self.detect_face_bbox(frame) | |
| if bbox is None: | |
| writer.write(frame) | |
| frame_idx += 1 | |
| continue | |
| face_crop = frame[bbox[1]:bbox[3], bbox[0]:bbox[2]] | |
| # Get audio features for this frame | |
| center = int(frame_idx * whisper_fps / original_fps) | |
| start = max(0, center - audio_context_window) | |
| end = min(all_audio_features.shape[1], center + audio_context_window) | |
| audio_segment = all_audio_features[:, start:end, :] | |
| if audio_segment.shape[1] == 0: | |
| audio_segment = torch.zeros(1, 1, 384, device=self.device) | |
| # Process | |
| processed_face = self.process_frame(face_crop, audio_segment, ref_latent) | |
| # Paste back | |
| result = self.paste_face(frame, processed_face, bbox) | |
| writer.write(result) | |
| frame_idx += 1 | |
| cap.release() | |
| writer.release() | |
| # Combine with audio using ffmpeg | |
| print("Combining with audio...") | |
| import shutil, subprocess | |
| ffmpeg = shutil.which("ffmpeg") | |
| if not ffmpeg: | |
| try: | |
| import imageio_ffmpeg | |
| ffmpeg = imageio_ffmpeg.get_ffmpeg_exe() | |
| except ImportError: | |
| ffmpeg = "ffmpeg" | |
| cmd = [ | |
| ffmpeg, "-y", | |
| "-i", output_path_temp, | |
| "-i", audio_path, | |
| "-c:v", "libx264", "-preset", "fast", | |
| "-c:a", "aac", "-b:a", "192k", | |
| "-shortest", | |
| output_path, | |
| "-loglevel", "error", | |
| ] | |
| try: | |
| subprocess.run(cmd, check=True, timeout=300) | |
| os.remove(output_path_temp) | |
| except Exception as e: | |
| print(f"FFmpeg failed: {e}. Output saved as {output_path_temp}") | |
| print(f"Output saved to: {output_path}") | |
| def main(): | |
| parser = argparse.ArgumentParser(description="LipSync Inference") | |
| parser.add_argument("--video", type=str, required=True, help="Input video path") | |
| parser.add_argument("--audio", type=str, required=True, help="Input audio path") | |
| parser.add_argument("--output", type=str, default="output.mp4", help="Output video path") | |
| parser.add_argument( | |
| "--checkpoint", | |
| type=str, | |
| default="checkpoints/best_model.pt", | |
| help="Model checkpoint", | |
| ) | |
| parser.add_argument("--device", type=str, default="cuda") | |
| parser.add_argument("--image-size", type=int, default=256) | |
| parser.add_argument("--no-ema", action="store_true", help="Don't use EMA weights") | |
| args = parser.parse_args() | |
| pipeline = LipSyncInference( | |
| checkpoint_path=args.checkpoint, | |
| device=args.device, | |
| image_size=args.image_size, | |
| use_ema=not args.no_ema, | |
| ) | |
| pipeline.run( | |
| video_path=args.video, | |
| audio_path=args.audio, | |
| output_path=args.output, | |
| ) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 14.7 kB
- Xet hash:
- 56c17f39d9913531af5527a42ce5bd7f5898001b8fc5937a6d6b1c1f4680e3d9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.