minhluanquach's picture
download
raw
14.5 kB
"""
Video Preprocessing Pipeline
==============================
Processes raw videos into training-ready format:
1. Extract frames at target FPS
2. Detect faces using MediaPipe Face Mesh
3. Crop and align face regions
4. Extract audio at 16kHz
5. Save metadata (face bboxes, landmarks)
Usage:
python scripts/preprocess.py --input data/raw --output data/processed --image-size 256
"""
import argparse
import json
import os
from pathlib import Path
from typing import Optional
import cv2
import mediapipe as mp
import numpy as np
import soundfile as sf
from tqdm import tqdm
# Resolve the MediaPipe face landmarker model path
_MODEL_DIR = Path(__file__).resolve().parent.parent / "models"
_FACE_LANDMARKER_PATH = _MODEL_DIR / "face_landmarker.task"
class FaceProcessor:
"""MediaPipe-based face detection and alignment (Tasks API >= 0.10.14)."""
def __init__(self, min_detection_confidence: float = 0.5):
if not _FACE_LANDMARKER_PATH.exists():
raise FileNotFoundError(
f"MediaPipe model not found at {_FACE_LANDMARKER_PATH}. "
"Download it with:\n"
' Invoke-WebRequest -Uri "https://storage.googleapis.com/mediapipe-models/'
'face_landmarker/face_landmarker/float16/latest/face_landmarker.task" '
'-OutFile "models/face_landmarker.task"'
)
base_options = mp.tasks.BaseOptions(
model_asset_path=str(_FACE_LANDMARKER_PATH)
)
options = mp.tasks.vision.FaceLandmarkerOptions(
base_options=base_options,
running_mode=mp.tasks.vision.RunningMode.IMAGE,
num_faces=1,
min_face_detection_confidence=min_detection_confidence,
min_face_presence_confidence=min_detection_confidence,
output_face_blendshapes=False,
output_facial_transformation_matrixes=False,
)
self.landmarker = mp.tasks.vision.FaceLandmarker.create_from_options(options)
# Lip region landmark indices (same indices as legacy face mesh)
self.LIPS_OUTER = [61, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291,
308, 324, 318, 402, 317, 14, 87, 178, 88, 95, 78]
def detect_face(self, image: np.ndarray) -> Optional[dict]:
"""Detect face and return landmarks + bounding box."""
rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
result = self.landmarker.detect(mp_image)
if not result.face_landmarks:
return None
landmarks = result.face_landmarks[0] # first face
h, w = image.shape[:2]
# Convert normalized landmarks to pixel coordinates
points = np.array(
[(lm.x * w, lm.y * h) for lm in landmarks],
dtype=np.float32,
)
# Compute bounding box with padding
x_min, y_min = points.min(axis=0)
x_max, y_max = points.max(axis=0)
# Add padding (20% of face size)
face_w = x_max - x_min
face_h = y_max - y_min
pad_w = face_w * 0.2
pad_h = face_h * 0.2
x_min = max(0, x_min - pad_w)
y_min = max(0, y_min - pad_h)
x_max = min(w, x_max + pad_w)
y_max = min(h, y_max + pad_h)
# Make square bbox
cx = (x_min + x_max) / 2
cy = (y_min + y_max) / 2
size = max(x_max - x_min, y_max - y_min)
half = size / 2
x_min = max(0, cx - half)
y_min = max(0, cy - half)
x_max = min(w, cx + half)
y_max = min(h, cy + half)
# Lip landmarks for SyncNet
lip_indices = [i for i in self.LIPS_OUTER if i < len(points)]
lip_points = points[lip_indices] if lip_indices else points[-22:]
return {
"bbox": [float(x_min), float(y_min), float(x_max), float(y_max)],
"landmarks": points.tolist(),
"lip_landmarks": lip_points.tolist(),
"face_size": float(size),
}
def crop_face(
self,
image: np.ndarray,
bbox: list[float],
target_size: int = 256,
) -> np.ndarray:
"""Crop and resize face region."""
x1, y1, x2, y2 = [int(c) for c in bbox]
h, w = image.shape[:2]
# Handle cases where bbox goes outside image
x1, y1 = max(0, x1), max(0, y1)
x2, y2 = min(w, x2), min(h, y2)
face = image[y1:y2, x1:x2]
if face.size == 0:
return None
face = cv2.resize(face, (target_size, target_size), interpolation=cv2.INTER_LANCZOS4)
return face
def get_lip_crop(
self,
image: np.ndarray,
landmarks: list,
target_size: int = 96,
) -> Optional[np.ndarray]:
"""Extract lip region crop for SyncNet training."""
lip_pts = np.array(landmarks, dtype=np.float32)
if len(lip_pts) == 0:
return None
# Compute lip bounding box
x_min, y_min = lip_pts.min(axis=0)
x_max, y_max = lip_pts.max(axis=0)
# Add padding
lip_w = x_max - x_min
lip_h = y_max - y_min
pad = max(lip_w, lip_h) * 0.5
cx = (x_min + x_max) / 2
cy = (y_min + y_max) / 2
size = max(lip_w, lip_h) + 2 * pad
half = size / 2
h, w = image.shape[:2]
x1 = max(0, int(cx - half))
y1 = max(0, int(cy - half))
x2 = min(w, int(cx + half))
y2 = min(h, int(cy + half))
lip_crop = image[y1:y2, x1:x2]
if lip_crop.size == 0:
return None
lip_crop = cv2.resize(lip_crop, (target_size, target_size), interpolation=cv2.INTER_LANCZOS4)
return lip_crop
def close(self):
self.landmarker.close()
def _find_ffmpeg() -> str:
"""Find ffmpeg executable, searching PATH and common install locations."""
import shutil
path = shutil.which("ffmpeg")
if path:
return path
# Common Windows install locations
import glob
for pattern in [
os.path.expandvars(r"%LOCALAPPDATA%\Microsoft\WinGet\Packages\*FFmpeg*\**\ffmpeg.exe"),
r"C:\ffmpeg\bin\ffmpeg.exe",
r"C:\Program Files\ffmpeg\bin\ffmpeg.exe",
]:
matches = glob.glob(pattern, recursive=True)
if matches:
return matches[0]
return "ffmpeg" # Hope it's on PATH at runtime
_FFMPEG_PATH: str | None = None
def _get_ffmpeg() -> str:
global _FFMPEG_PATH
if _FFMPEG_PATH is None:
_FFMPEG_PATH = _find_ffmpeg()
return _FFMPEG_PATH
def extract_audio(video_path: Path, output_path: Path, sr: int = 16000) -> bool:
"""Extract audio from video using ffmpeg.
Falls back to torchaudio if ffmpeg is not available.
"""
import subprocess
ffmpeg = _get_ffmpeg()
try:
cmd = [
ffmpeg, "-y", "-i", str(video_path),
"-vn", "-acodec", "pcm_s16le",
"-ar", str(sr), "-ac", "1",
str(output_path),
"-loglevel", "error",
]
subprocess.run(cmd, check=True, capture_output=True, timeout=120)
return True
except FileNotFoundError:
# ffmpeg binary not found — try torchaudio as fallback
try:
import torchaudio
waveform, orig_sr = torchaudio.load(str(video_path))
if orig_sr != sr:
waveform = torchaudio.functional.resample(waveform, orig_sr, sr)
if waveform.shape[0] > 1:
waveform = waveform.mean(dim=0, keepdim=True)
torchaudio.save(str(output_path), waveform, sr)
return True
except Exception as e2:
print(f"Audio extraction failed for {video_path} (no ffmpeg, torchaudio fallback failed): {e2}")
return False
except Exception as e:
print(f"Audio extraction failed for {video_path}: {e}")
return False
def process_video(
video_path: Path,
output_dir: Path,
face_processor: FaceProcessor,
target_fps: int = 25,
image_size: int = 256,
lip_size: int = 96,
) -> Optional[dict]:
"""Process a single video into frames + audio + metadata."""
video_name = video_path.stem
# Create output directories
frames_dir = output_dir / video_name / "frames"
lips_dir = output_dir / video_name / "lips"
frames_dir.mkdir(parents=True, exist_ok=True)
lips_dir.mkdir(parents=True, exist_ok=True)
# Extract audio
audio_path = output_dir / video_name / "audio.wav"
if not audio_path.exists():
# Check if video has audio or if audio file exists alongside video
audio_source = video_path.with_suffix(".wav")
if audio_source.exists():
# Copy pre-extracted audio
import shutil
shutil.copy2(audio_source, audio_path)
else:
if not extract_audio(video_path, audio_path):
return None
# Open video
cap = cv2.VideoCapture(str(video_path))
if not cap.isOpened():
print(f"Cannot open video: {video_path}")
return None
original_fps = cap.get(cv2.CAP_PROP_FPS)
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
if original_fps <= 0:
original_fps = 25.0
# Frame sampling rate
frame_interval = max(1, round(original_fps / target_fps))
metadata = {
"video_name": video_name,
"original_fps": original_fps,
"target_fps": target_fps,
"image_size": image_size,
"frames": [],
}
frame_idx = 0
saved_idx = 0
consecutive_failures = 0
max_consecutive_failures = 150 # Skip video if ~5s of frames have no face
while True:
ret, frame = cap.read()
if not ret:
break
if frame_idx % frame_interval != 0:
frame_idx += 1
continue
# Detect face
face_info = face_processor.detect_face(frame)
if face_info is None:
consecutive_failures += 1
if consecutive_failures > max_consecutive_failures:
break
frame_idx += 1
continue
consecutive_failures = 0
# Crop face
face_crop = face_processor.crop_face(frame, face_info["bbox"], image_size)
if face_crop is None:
frame_idx += 1
continue
# Crop lips
lip_crop = face_processor.get_lip_crop(
frame, face_info["lip_landmarks"], lip_size
)
# Save face frame
frame_path = frames_dir / f"{saved_idx:06d}.png"
cv2.imwrite(str(frame_path), face_crop)
# Save lip crop (for SyncNet)
if lip_crop is not None:
lip_path = lips_dir / f"{saved_idx:06d}.png"
cv2.imwrite(str(lip_path), lip_crop)
# Store metadata
frame_meta = {
"frame_idx": saved_idx,
"original_frame_idx": frame_idx,
"timestamp": frame_idx / original_fps,
"bbox": face_info["bbox"],
"face_size": face_info["face_size"],
}
metadata["frames"].append(frame_meta)
saved_idx += 1
frame_idx += 1
cap.release()
if saved_idx < 10:
print(f"Too few valid frames ({saved_idx}) for {video_name}, skipping")
return None
metadata["total_frames"] = saved_idx
metadata["duration"] = saved_idx / target_fps
# Save metadata
meta_path = output_dir / video_name / "metadata.json"
with open(meta_path, "w") as f:
json.dump(metadata, f, indent=2)
return metadata
def main():
parser = argparse.ArgumentParser(description="Preprocess videos for lipsync training")
parser.add_argument("--input", type=str, required=True, help="Input video directory")
parser.add_argument("--output", type=str, required=True, help="Output processed directory")
parser.add_argument("--image-size", type=int, default=256, help="Face crop size")
parser.add_argument("--lip-size", type=int, default=96, help="Lip crop size for SyncNet")
parser.add_argument("--fps", type=int, default=25, help="Target FPS")
parser.add_argument(
"--min-confidence",
type=float,
default=0.5,
help="Face detection confidence threshold",
)
args = parser.parse_args()
input_dir = Path(args.input)
output_dir = Path(args.output)
output_dir.mkdir(parents=True, exist_ok=True)
# Find all video files
video_extensions = {".mp4", ".avi", ".mov", ".mkv", ".webm"}
video_files = [
f
for f in sorted(input_dir.iterdir())
if f.suffix.lower() in video_extensions
]
if not video_files:
print(f"No video files found in {input_dir}")
return
print(f"Found {len(video_files)} videos to process")
face_processor = FaceProcessor(min_detection_confidence=args.min_confidence)
all_metadata = []
for video_path in tqdm(video_files, desc="Processing videos"):
meta = process_video(
video_path=video_path,
output_dir=output_dir,
face_processor=face_processor,
target_fps=args.fps,
image_size=args.image_size,
lip_size=args.lip_size,
)
if meta:
all_metadata.append(meta)
face_processor.close()
# Save global metadata
global_meta = {
"total_videos": len(all_metadata),
"total_frames": sum(m["total_frames"] for m in all_metadata),
"total_duration_hours": sum(m["duration"] for m in all_metadata) / 3600,
"videos": [m["video_name"] for m in all_metadata],
}
global_meta_path = output_dir / "dataset_info.json"
with open(global_meta_path, "w") as f:
json.dump(global_meta, f, indent=2)
print(f"\nProcessing complete:")
print(f" Videos processed: {global_meta['total_videos']}")
print(f" Total frames: {global_meta['total_frames']}")
print(f" Total duration: {global_meta['total_duration_hours']:.2f} hours")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
14.5 kB
·
Xet hash:
fcb68f93107e9ef3869b29550f55ece24038220691687fafaff48d7587487b56

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.