minhluanquach's picture
download
raw
14.7 kB
"""
Inference Pipeline
====================
Takes a video + audio and produces a lip-synced output video.
Pipeline:
1. Extract faces from video frames
2. Extract Whisper features from audio
3. For each frame: encode → mask → fuse with audio → decode
4. Paste generated face back onto original frame
5. Combine frames into output video with original audio
Usage:
python -m src.inference.predict \
--video input.mp4 \
--audio speech.wav \
--checkpoint checkpoints/best_model.pt \
--output result.mp4
"""
import argparse
import os
import tempfile
from pathlib import Path
import cv2
import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image
from torchvision import transforms
from tqdm import tqdm
from src.models.unet import LipSyncUNet
class LipSyncInference:
"""Complete inference pipeline for lip synchronization."""
def __init__(
self,
checkpoint_path: str,
vae_model_id: str = "stable-diffusion-v1-5/stable-diffusion-v1-5",
whisper_model_id: str = "openai/whisper-tiny",
device: str = "cuda",
image_size: int = 256,
use_ema: bool = True,
):
self.device = torch.device(device if torch.cuda.is_available() else "cpu")
self.image_size = image_size
print(f"Using device: {self.device}")
# Load UNet
print("Loading UNet...")
ckpt = torch.load(checkpoint_path, map_location=self.device, weights_only=False)
unet_config = ckpt["config"]["model"]["unet"]
self.unet = LipSyncUNet(**unet_config).to(self.device)
# Load EMA weights if available
if use_ema and "ema_state_dict" in ckpt:
print("Using EMA weights")
ema_shadow = ckpt["ema_state_dict"]["shadow"]
state_dict = self.unet.state_dict()
for name in ema_shadow:
if name in state_dict:
state_dict[name] = ema_shadow[name]
self.unet.load_state_dict(state_dict)
else:
self.unet.load_state_dict(ckpt["model_state_dict"])
self.unet.eval()
self.unet.requires_grad_(False)
# Load VAE
print("Loading VAE...")
from diffusers import AutoencoderKL
self.vae = AutoencoderKL.from_pretrained(
vae_model_id, subfolder="vae"
).to(self.device)
self.vae.eval()
self.vae.requires_grad_(False)
self.vae_scale = self.vae.config.scaling_factor
# Load Whisper
print("Loading Whisper...")
from transformers import WhisperModel, WhisperProcessor
self.whisper_processor = WhisperProcessor.from_pretrained(whisper_model_id)
self.whisper = WhisperModel.from_pretrained(whisper_model_id).to(self.device)
self.whisper.eval()
self.whisper.requires_grad_(False)
# Face detector
print("Loading face detector...")
import mediapipe as mp
model_path = Path(__file__).resolve().parent.parent.parent / "models" / "face_landmarker.task"
base_options = mp.tasks.BaseOptions(model_asset_path=str(model_path))
options = mp.tasks.vision.FaceLandmarkerOptions(
base_options=base_options,
running_mode=mp.tasks.vision.RunningMode.IMAGE,
num_faces=1,
min_face_detection_confidence=0.5,
min_face_presence_confidence=0.5,
)
self.face_landmarker = mp.tasks.vision.FaceLandmarker.create_from_options(options)
# Image transform
self.transform = transforms.Compose([
transforms.Resize(image_size),
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
])
print("Inference pipeline ready!")
def detect_face_bbox(self, frame: np.ndarray) -> list[int]:
"""Detect face bounding box in a frame."""
import mediapipe as mp
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
result = self.face_landmarker.detect(mp_image)
if not result.face_landmarks:
return None
landmarks = result.face_landmarks[0]
h, w = frame.shape[:2]
points = np.array([(lm.x * w, lm.y * h) for lm in landmarks])
x_min, y_min = points.min(axis=0)
x_max, y_max = points.max(axis=0)
face_w = x_max - x_min
face_h = y_max - y_min
pad = max(face_w, face_h) * 0.2
cx = (x_min + x_max) / 2
cy = (y_min + y_max) / 2
size = max(face_w, face_h) + 2 * pad
half = size / 2
x1 = int(max(0, cx - half))
y1 = int(max(0, cy - half))
x2 = int(min(w, cx + half))
y2 = int(min(h, cy + half))
return [x1, y1, x2, y2]
def extract_whisper_features(self, audio_path: str) -> torch.Tensor:
"""Extract Whisper encoder features from audio file."""
import librosa
audio, sr = librosa.load(audio_path, sr=16000, mono=True)
# Process in 30s chunks
chunk_size = 30 * sr
all_features = []
for start in range(0, len(audio), chunk_size):
chunk = audio[start:start + chunk_size]
if len(chunk) < sr:
continue
inputs = self.whisper_processor(chunk, sampling_rate=sr, return_tensors="pt")
input_features = inputs.input_features.to(self.device)
with torch.no_grad():
features = self.whisper.encoder(input_features).last_hidden_state
all_features.append(features.cpu())
if not all_features:
return torch.zeros(1, 1, 384).to(self.device)
return torch.cat(all_features, dim=1).to(self.device)
@torch.no_grad()
def process_frame(
self,
face_crop: np.ndarray,
audio_features: torch.Tensor,
reference_latent: torch.Tensor,
) -> np.ndarray:
"""Process a single face crop with audio features."""
# Prep image
img = Image.fromarray(cv2.cvtColor(face_crop, cv2.COLOR_BGR2RGB))
img_tensor = self.transform(img).unsqueeze(0).to(self.device)
# Encode to latent
latent = self.vae.encode(img_tensor).latent_dist.sample() * self.vae_scale
# Create mask (lower half)
B, C, H, W = latent.shape
mask = torch.zeros(B, 1, H, W, device=self.device)
mask[:, :, H // 2:, :] = 1.0
masked = latent * (1 - mask)
# UNet prediction
pred_latent = self.unet(
masked_latent=masked,
reference_latent=reference_latent,
mask=mask,
audio_features=audio_features,
)
# Decode
decoded = self.vae.decode(pred_latent / self.vae_scale).sample
decoded = decoded.clamp(-1, 1)
# Blend: keep original upper half, use predicted lower half
# This eliminates the visible mask seam from latent-space masking
pixel_h = img_tensor.shape[2]
blend_start = pixel_h // 2
feather = max(4, pixel_h // 32)
blended = img_tensor.clone()
blended[:, :, blend_start:, :] = decoded[:, :, blend_start:, :]
for i in range(feather):
alpha = i / feather
row = blend_start - feather + i
if 0 <= row < pixel_h:
blended[:, :, row, :] = (
(1 - alpha) * img_tensor[:, :, row, :]
+ alpha * decoded[:, :, row, :]
)
# Convert to numpy image
img_out = ((blended[0].permute(1, 2, 0).cpu().numpy() + 1) * 127.5).astype(np.uint8)
img_out = cv2.cvtColor(img_out, cv2.COLOR_RGB2BGR)
img_out = cv2.resize(img_out, (face_crop.shape[1], face_crop.shape[0]))
return img_out
def paste_face(
self,
frame: np.ndarray,
face: np.ndarray,
bbox: list[int],
) -> np.ndarray:
"""Paste generated face back onto original frame with blending."""
x1, y1, x2, y2 = bbox
h, w = y2 - y1, x2 - x1
face_resized = cv2.resize(face, (w, h))
result = frame.copy()
# Create a smooth blending mask (feathered edges)
blend_mask = np.ones((h, w), dtype=np.float32)
feather = min(h, w) // 10
if feather > 0:
for i in range(feather):
alpha = i / feather
blend_mask[i, :] *= alpha
blend_mask[h - 1 - i, :] *= alpha
blend_mask[:, i] *= alpha
blend_mask[:, w - 1 - i] *= alpha
blend_mask = blend_mask[:, :, np.newaxis]
# Only blend the lower half (where lips are modified)
mid_y = h // 2
blend_mask[:mid_y] = 0 # Don't change upper half
result[y1:y2, x1:x2] = (
result[y1:y2, x1:x2] * (1 - blend_mask)
+ face_resized * blend_mask
).astype(np.uint8)
return result
def run(
self,
video_path: str,
audio_path: str,
output_path: str,
fps: int = 25,
batch_size: int = 4,
):
"""Run full inference pipeline."""
print(f"\nProcessing video: {video_path}")
print(f"Audio: {audio_path}")
# Extract audio features
print("Extracting audio features...")
all_audio_features = self.extract_whisper_features(audio_path)
# Open video
cap = cv2.VideoCapture(video_path)
original_fps = cap.get(cv2.CAP_PROP_FPS) or fps
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
frame_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
frame_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
# Output writer
output_path_temp = output_path + ".temp.mp4"
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
writer = cv2.VideoWriter(output_path_temp, fourcc, original_fps, (frame_w, frame_h))
# Get reference frame (first frame with detectable face)
ref_latent = None
cap_ref = cv2.VideoCapture(video_path)
for _ in range(min(30, total_frames)):
ret, frame = cap_ref.read()
if not ret:
break
bbox = self.detect_face_bbox(frame)
if bbox:
face_crop = frame[bbox[1]:bbox[3], bbox[0]:bbox[2]]
face_crop = cv2.resize(face_crop, (self.image_size, self.image_size))
img = Image.fromarray(cv2.cvtColor(face_crop, cv2.COLOR_BGR2RGB))
img_t = self.transform(img).unsqueeze(0).to(self.device)
ref_latent = self.vae.encode(img_t).latent_dist.sample() * self.vae_scale
break
cap_ref.release()
if ref_latent is None:
print("ERROR: Could not detect face in video for reference")
cap.release()
return
# Process frames
whisper_fps = 1500 / 30 # 50 features/sec
# Half-window must match training: num_audio_frames * 4 / 2 = 10
audio_context_window = 10
print(f"Processing {total_frames} frames...")
frame_idx = 0
for _ in tqdm(range(total_frames), desc="Generating"):
ret, frame = cap.read()
if not ret:
break
bbox = self.detect_face_bbox(frame)
if bbox is None:
writer.write(frame)
frame_idx += 1
continue
face_crop = frame[bbox[1]:bbox[3], bbox[0]:bbox[2]]
# Get audio features for this frame
center = int(frame_idx * whisper_fps / original_fps)
start = max(0, center - audio_context_window)
end = min(all_audio_features.shape[1], center + audio_context_window)
audio_segment = all_audio_features[:, start:end, :]
if audio_segment.shape[1] == 0:
audio_segment = torch.zeros(1, 1, 384, device=self.device)
# Process
processed_face = self.process_frame(face_crop, audio_segment, ref_latent)
# Paste back
result = self.paste_face(frame, processed_face, bbox)
writer.write(result)
frame_idx += 1
cap.release()
writer.release()
# Combine with audio using ffmpeg
print("Combining with audio...")
import shutil, subprocess
ffmpeg = shutil.which("ffmpeg")
if not ffmpeg:
try:
import imageio_ffmpeg
ffmpeg = imageio_ffmpeg.get_ffmpeg_exe()
except ImportError:
ffmpeg = "ffmpeg"
cmd = [
ffmpeg, "-y",
"-i", output_path_temp,
"-i", audio_path,
"-c:v", "libx264", "-preset", "fast",
"-c:a", "aac", "-b:a", "192k",
"-shortest",
output_path,
"-loglevel", "error",
]
try:
subprocess.run(cmd, check=True, timeout=300)
os.remove(output_path_temp)
except Exception as e:
print(f"FFmpeg failed: {e}. Output saved as {output_path_temp}")
print(f"Output saved to: {output_path}")
def main():
parser = argparse.ArgumentParser(description="LipSync Inference")
parser.add_argument("--video", type=str, required=True, help="Input video path")
parser.add_argument("--audio", type=str, required=True, help="Input audio path")
parser.add_argument("--output", type=str, default="output.mp4", help="Output video path")
parser.add_argument(
"--checkpoint",
type=str,
default="checkpoints/best_model.pt",
help="Model checkpoint",
)
parser.add_argument("--device", type=str, default="cuda")
parser.add_argument("--image-size", type=int, default=256)
parser.add_argument("--no-ema", action="store_true", help="Don't use EMA weights")
args = parser.parse_args()
pipeline = LipSyncInference(
checkpoint_path=args.checkpoint,
device=args.device,
image_size=args.image_size,
use_ema=not args.no_ema,
)
pipeline.run(
video_path=args.video,
audio_path=args.audio,
output_path=args.output,
)
if __name__ == "__main__":
main()

Xet Storage Details

Size:
14.7 kB
·
Xet hash:
56c17f39d9913531af5527a42ce5bd7f5898001b8fc5937a6d6b1c1f4680e3d9

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.