Buckets:
| """ | |
| Video Preprocessing Pipeline | |
| ============================== | |
| Processes raw videos into training-ready format: | |
| 1. Extract frames at target FPS | |
| 2. Detect faces using MediaPipe Face Mesh | |
| 3. Crop and align face regions | |
| 4. Extract audio at 16kHz | |
| 5. Save metadata (face bboxes, landmarks) | |
| Usage: | |
| python scripts/preprocess.py --input data/raw --output data/processed --image-size 256 | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| from pathlib import Path | |
| from typing import Optional | |
| import cv2 | |
| import mediapipe as mp | |
| import numpy as np | |
| import soundfile as sf | |
| from tqdm import tqdm | |
| # Resolve the MediaPipe face landmarker model path | |
| _MODEL_DIR = Path(__file__).resolve().parent.parent / "models" | |
| _FACE_LANDMARKER_PATH = _MODEL_DIR / "face_landmarker.task" | |
| class FaceProcessor: | |
| """MediaPipe-based face detection and alignment (Tasks API >= 0.10.14).""" | |
| def __init__(self, min_detection_confidence: float = 0.5): | |
| if not _FACE_LANDMARKER_PATH.exists(): | |
| raise FileNotFoundError( | |
| f"MediaPipe model not found at {_FACE_LANDMARKER_PATH}. " | |
| "Download it with:\n" | |
| ' Invoke-WebRequest -Uri "https://storage.googleapis.com/mediapipe-models/' | |
| 'face_landmarker/face_landmarker/float16/latest/face_landmarker.task" ' | |
| '-OutFile "models/face_landmarker.task"' | |
| ) | |
| base_options = mp.tasks.BaseOptions( | |
| model_asset_path=str(_FACE_LANDMARKER_PATH) | |
| ) | |
| options = mp.tasks.vision.FaceLandmarkerOptions( | |
| base_options=base_options, | |
| running_mode=mp.tasks.vision.RunningMode.IMAGE, | |
| num_faces=1, | |
| min_face_detection_confidence=min_detection_confidence, | |
| min_face_presence_confidence=min_detection_confidence, | |
| output_face_blendshapes=False, | |
| output_facial_transformation_matrixes=False, | |
| ) | |
| self.landmarker = mp.tasks.vision.FaceLandmarker.create_from_options(options) | |
| # Lip region landmark indices (same indices as legacy face mesh) | |
| self.LIPS_OUTER = [61, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291, | |
| 308, 324, 318, 402, 317, 14, 87, 178, 88, 95, 78] | |
| def detect_face(self, image: np.ndarray) -> Optional[dict]: | |
| """Detect face and return landmarks + bounding box.""" | |
| rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) | |
| mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb) | |
| result = self.landmarker.detect(mp_image) | |
| if not result.face_landmarks: | |
| return None | |
| landmarks = result.face_landmarks[0] # first face | |
| h, w = image.shape[:2] | |
| # Convert normalized landmarks to pixel coordinates | |
| points = np.array( | |
| [(lm.x * w, lm.y * h) for lm in landmarks], | |
| dtype=np.float32, | |
| ) | |
| # Compute bounding box with padding | |
| x_min, y_min = points.min(axis=0) | |
| x_max, y_max = points.max(axis=0) | |
| # Add padding (20% of face size) | |
| face_w = x_max - x_min | |
| face_h = y_max - y_min | |
| pad_w = face_w * 0.2 | |
| pad_h = face_h * 0.2 | |
| x_min = max(0, x_min - pad_w) | |
| y_min = max(0, y_min - pad_h) | |
| x_max = min(w, x_max + pad_w) | |
| y_max = min(h, y_max + pad_h) | |
| # Make square bbox | |
| cx = (x_min + x_max) / 2 | |
| cy = (y_min + y_max) / 2 | |
| size = max(x_max - x_min, y_max - y_min) | |
| half = size / 2 | |
| x_min = max(0, cx - half) | |
| y_min = max(0, cy - half) | |
| x_max = min(w, cx + half) | |
| y_max = min(h, cy + half) | |
| # Lip landmarks for SyncNet | |
| lip_indices = [i for i in self.LIPS_OUTER if i < len(points)] | |
| lip_points = points[lip_indices] if lip_indices else points[-22:] | |
| return { | |
| "bbox": [float(x_min), float(y_min), float(x_max), float(y_max)], | |
| "landmarks": points.tolist(), | |
| "lip_landmarks": lip_points.tolist(), | |
| "face_size": float(size), | |
| } | |
| def crop_face( | |
| self, | |
| image: np.ndarray, | |
| bbox: list[float], | |
| target_size: int = 256, | |
| ) -> np.ndarray: | |
| """Crop and resize face region.""" | |
| x1, y1, x2, y2 = [int(c) for c in bbox] | |
| h, w = image.shape[:2] | |
| # Handle cases where bbox goes outside image | |
| x1, y1 = max(0, x1), max(0, y1) | |
| x2, y2 = min(w, x2), min(h, y2) | |
| face = image[y1:y2, x1:x2] | |
| if face.size == 0: | |
| return None | |
| face = cv2.resize(face, (target_size, target_size), interpolation=cv2.INTER_LANCZOS4) | |
| return face | |
| def get_lip_crop( | |
| self, | |
| image: np.ndarray, | |
| landmarks: list, | |
| target_size: int = 96, | |
| ) -> Optional[np.ndarray]: | |
| """Extract lip region crop for SyncNet training.""" | |
| lip_pts = np.array(landmarks, dtype=np.float32) | |
| if len(lip_pts) == 0: | |
| return None | |
| # Compute lip bounding box | |
| x_min, y_min = lip_pts.min(axis=0) | |
| x_max, y_max = lip_pts.max(axis=0) | |
| # Add padding | |
| lip_w = x_max - x_min | |
| lip_h = y_max - y_min | |
| pad = max(lip_w, lip_h) * 0.5 | |
| cx = (x_min + x_max) / 2 | |
| cy = (y_min + y_max) / 2 | |
| size = max(lip_w, lip_h) + 2 * pad | |
| half = size / 2 | |
| h, w = image.shape[:2] | |
| x1 = max(0, int(cx - half)) | |
| y1 = max(0, int(cy - half)) | |
| x2 = min(w, int(cx + half)) | |
| y2 = min(h, int(cy + half)) | |
| lip_crop = image[y1:y2, x1:x2] | |
| if lip_crop.size == 0: | |
| return None | |
| lip_crop = cv2.resize(lip_crop, (target_size, target_size), interpolation=cv2.INTER_LANCZOS4) | |
| return lip_crop | |
| def close(self): | |
| self.landmarker.close() | |
| def _find_ffmpeg() -> str: | |
| """Find ffmpeg executable, searching PATH and common install locations.""" | |
| import shutil | |
| path = shutil.which("ffmpeg") | |
| if path: | |
| return path | |
| # Common Windows install locations | |
| import glob | |
| for pattern in [ | |
| os.path.expandvars(r"%LOCALAPPDATA%\Microsoft\WinGet\Packages\*FFmpeg*\**\ffmpeg.exe"), | |
| r"C:\ffmpeg\bin\ffmpeg.exe", | |
| r"C:\Program Files\ffmpeg\bin\ffmpeg.exe", | |
| ]: | |
| matches = glob.glob(pattern, recursive=True) | |
| if matches: | |
| return matches[0] | |
| return "ffmpeg" # Hope it's on PATH at runtime | |
| _FFMPEG_PATH: str | None = None | |
| def _get_ffmpeg() -> str: | |
| global _FFMPEG_PATH | |
| if _FFMPEG_PATH is None: | |
| _FFMPEG_PATH = _find_ffmpeg() | |
| return _FFMPEG_PATH | |
| def extract_audio(video_path: Path, output_path: Path, sr: int = 16000) -> bool: | |
| """Extract audio from video using ffmpeg. | |
| Falls back to torchaudio if ffmpeg is not available. | |
| """ | |
| import subprocess | |
| ffmpeg = _get_ffmpeg() | |
| try: | |
| cmd = [ | |
| ffmpeg, "-y", "-i", str(video_path), | |
| "-vn", "-acodec", "pcm_s16le", | |
| "-ar", str(sr), "-ac", "1", | |
| str(output_path), | |
| "-loglevel", "error", | |
| ] | |
| subprocess.run(cmd, check=True, capture_output=True, timeout=120) | |
| return True | |
| except FileNotFoundError: | |
| # ffmpeg binary not found — try torchaudio as fallback | |
| try: | |
| import torchaudio | |
| waveform, orig_sr = torchaudio.load(str(video_path)) | |
| if orig_sr != sr: | |
| waveform = torchaudio.functional.resample(waveform, orig_sr, sr) | |
| if waveform.shape[0] > 1: | |
| waveform = waveform.mean(dim=0, keepdim=True) | |
| torchaudio.save(str(output_path), waveform, sr) | |
| return True | |
| except Exception as e2: | |
| print(f"Audio extraction failed for {video_path} (no ffmpeg, torchaudio fallback failed): {e2}") | |
| return False | |
| except Exception as e: | |
| print(f"Audio extraction failed for {video_path}: {e}") | |
| return False | |
| def process_video( | |
| video_path: Path, | |
| output_dir: Path, | |
| face_processor: FaceProcessor, | |
| target_fps: int = 25, | |
| image_size: int = 256, | |
| lip_size: int = 96, | |
| ) -> Optional[dict]: | |
| """Process a single video into frames + audio + metadata.""" | |
| video_name = video_path.stem | |
| # Create output directories | |
| frames_dir = output_dir / video_name / "frames" | |
| lips_dir = output_dir / video_name / "lips" | |
| frames_dir.mkdir(parents=True, exist_ok=True) | |
| lips_dir.mkdir(parents=True, exist_ok=True) | |
| # Extract audio | |
| audio_path = output_dir / video_name / "audio.wav" | |
| if not audio_path.exists(): | |
| # Check if video has audio or if audio file exists alongside video | |
| audio_source = video_path.with_suffix(".wav") | |
| if audio_source.exists(): | |
| # Copy pre-extracted audio | |
| import shutil | |
| shutil.copy2(audio_source, audio_path) | |
| else: | |
| if not extract_audio(video_path, audio_path): | |
| return None | |
| # Open video | |
| cap = cv2.VideoCapture(str(video_path)) | |
| if not cap.isOpened(): | |
| print(f"Cannot open video: {video_path}") | |
| return None | |
| original_fps = cap.get(cv2.CAP_PROP_FPS) | |
| total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) | |
| if original_fps <= 0: | |
| original_fps = 25.0 | |
| # Frame sampling rate | |
| frame_interval = max(1, round(original_fps / target_fps)) | |
| metadata = { | |
| "video_name": video_name, | |
| "original_fps": original_fps, | |
| "target_fps": target_fps, | |
| "image_size": image_size, | |
| "frames": [], | |
| } | |
| frame_idx = 0 | |
| saved_idx = 0 | |
| consecutive_failures = 0 | |
| max_consecutive_failures = 150 # Skip video if ~5s of frames have no face | |
| while True: | |
| ret, frame = cap.read() | |
| if not ret: | |
| break | |
| if frame_idx % frame_interval != 0: | |
| frame_idx += 1 | |
| continue | |
| # Detect face | |
| face_info = face_processor.detect_face(frame) | |
| if face_info is None: | |
| consecutive_failures += 1 | |
| if consecutive_failures > max_consecutive_failures: | |
| break | |
| frame_idx += 1 | |
| continue | |
| consecutive_failures = 0 | |
| # Crop face | |
| face_crop = face_processor.crop_face(frame, face_info["bbox"], image_size) | |
| if face_crop is None: | |
| frame_idx += 1 | |
| continue | |
| # Crop lips | |
| lip_crop = face_processor.get_lip_crop( | |
| frame, face_info["lip_landmarks"], lip_size | |
| ) | |
| # Save face frame | |
| frame_path = frames_dir / f"{saved_idx:06d}.png" | |
| cv2.imwrite(str(frame_path), face_crop) | |
| # Save lip crop (for SyncNet) | |
| if lip_crop is not None: | |
| lip_path = lips_dir / f"{saved_idx:06d}.png" | |
| cv2.imwrite(str(lip_path), lip_crop) | |
| # Store metadata | |
| frame_meta = { | |
| "frame_idx": saved_idx, | |
| "original_frame_idx": frame_idx, | |
| "timestamp": frame_idx / original_fps, | |
| "bbox": face_info["bbox"], | |
| "face_size": face_info["face_size"], | |
| } | |
| metadata["frames"].append(frame_meta) | |
| saved_idx += 1 | |
| frame_idx += 1 | |
| cap.release() | |
| if saved_idx < 10: | |
| print(f"Too few valid frames ({saved_idx}) for {video_name}, skipping") | |
| return None | |
| metadata["total_frames"] = saved_idx | |
| metadata["duration"] = saved_idx / target_fps | |
| # Save metadata | |
| meta_path = output_dir / video_name / "metadata.json" | |
| with open(meta_path, "w") as f: | |
| json.dump(metadata, f, indent=2) | |
| return metadata | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Preprocess videos for lipsync training") | |
| parser.add_argument("--input", type=str, required=True, help="Input video directory") | |
| parser.add_argument("--output", type=str, required=True, help="Output processed directory") | |
| parser.add_argument("--image-size", type=int, default=256, help="Face crop size") | |
| parser.add_argument("--lip-size", type=int, default=96, help="Lip crop size for SyncNet") | |
| parser.add_argument("--fps", type=int, default=25, help="Target FPS") | |
| parser.add_argument( | |
| "--min-confidence", | |
| type=float, | |
| default=0.5, | |
| help="Face detection confidence threshold", | |
| ) | |
| args = parser.parse_args() | |
| input_dir = Path(args.input) | |
| output_dir = Path(args.output) | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| # Find all video files | |
| video_extensions = {".mp4", ".avi", ".mov", ".mkv", ".webm"} | |
| video_files = [ | |
| f | |
| for f in sorted(input_dir.iterdir()) | |
| if f.suffix.lower() in video_extensions | |
| ] | |
| if not video_files: | |
| print(f"No video files found in {input_dir}") | |
| return | |
| print(f"Found {len(video_files)} videos to process") | |
| face_processor = FaceProcessor(min_detection_confidence=args.min_confidence) | |
| all_metadata = [] | |
| for video_path in tqdm(video_files, desc="Processing videos"): | |
| meta = process_video( | |
| video_path=video_path, | |
| output_dir=output_dir, | |
| face_processor=face_processor, | |
| target_fps=args.fps, | |
| image_size=args.image_size, | |
| lip_size=args.lip_size, | |
| ) | |
| if meta: | |
| all_metadata.append(meta) | |
| face_processor.close() | |
| # Save global metadata | |
| global_meta = { | |
| "total_videos": len(all_metadata), | |
| "total_frames": sum(m["total_frames"] for m in all_metadata), | |
| "total_duration_hours": sum(m["duration"] for m in all_metadata) / 3600, | |
| "videos": [m["video_name"] for m in all_metadata], | |
| } | |
| global_meta_path = output_dir / "dataset_info.json" | |
| with open(global_meta_path, "w") as f: | |
| json.dump(global_meta, f, indent=2) | |
| print(f"\nProcessing complete:") | |
| print(f" Videos processed: {global_meta['total_videos']}") | |
| print(f" Total frames: {global_meta['total_frames']}") | |
| print(f" Total duration: {global_meta['total_duration_hours']:.2f} hours") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 14.5 kB
- Xet hash:
- fcb68f93107e9ef3869b29550f55ece24038220691687fafaff48d7587487b56
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.