minhluanquach's picture
download
raw
11.2 kB
"""
Feature Extraction Script
==========================
Pre-computes and caches expensive features:
1. VAE latent encoding of face frames
2. Whisper audio features
3. Mel spectrograms for SyncNet
This avoids recomputing these on every training epoch.
Usage:
python scripts/extract_features.py --input data/processed --output data/cache
"""
import argparse
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import librosa
import numpy as np
import torch
import torch.nn.functional as F
from PIL import Image
from torchvision import transforms
from tqdm import tqdm
def extract_mel_spectrogram(
audio_path: Path,
sr: int = 16000,
n_fft: int = 1024,
hop_length: int = 256,
n_mels: int = 80,
) -> np.ndarray:
"""Extract mel spectrogram from audio file."""
audio, _ = librosa.load(str(audio_path), sr=sr, mono=True)
# Compute mel spectrogram
mel = librosa.feature.melspectrogram(
y=audio,
sr=sr,
n_fft=n_fft,
hop_length=hop_length,
n_mels=n_mels,
fmin=50,
fmax=7600,
)
# Log scale
mel = librosa.power_to_db(mel, ref=np.max)
# Normalize to [-1, 1]
mel = (mel - mel.min()) / (mel.max() - mel.min() + 1e-8) * 2 - 1
return mel.astype(np.float32)
def extract_whisper_features(
audio_path: Path,
whisper_model,
whisper_processor,
device: torch.device,
sr: int = 16000,
chunk_duration: float = 30.0,
use_fp16: bool = True,
) -> np.ndarray:
"""Extract Whisper encoder features from audio (GPU-accelerated, FP16)."""
audio, _ = librosa.load(str(audio_path), sr=sr, mono=True)
chunk_samples = int(chunk_duration * sr)
all_features = []
dtype = torch.float16 if (use_fp16 and device.type == "cuda") else torch.float32
for start in range(0, len(audio), chunk_samples):
chunk = audio[start : start + chunk_samples]
if len(chunk) < sr:
continue
inputs = whisper_processor(
chunk, sampling_rate=sr, return_tensors="pt",
)
input_features = inputs.input_features.to(device=device, dtype=dtype)
with torch.no_grad(), torch.amp.autocast(device_type=device.type, enabled=(device.type == "cuda")):
encoder_output = whisper_model.encoder(input_features)
features = encoder_output.last_hidden_state
all_features.append(features.float().cpu().numpy())
if not all_features:
return np.zeros((1, 1, 384), dtype=np.float32)
return np.concatenate(all_features, axis=1)
def _load_and_transform(args):
"""Load a single image and apply transform (for ThreadPool)."""
path, transform = args
img = Image.open(path).convert("RGB")
return transform(img)
def extract_vae_latents(
frames_dir: Path,
vae,
device: torch.device,
image_size: int = 256,
batch_size: int = 64,
num_workers: int = 4,
use_fp16: bool = True,
) -> np.ndarray:
"""Encode face frames into VAE latent space (GPU-accelerated, FP16, threaded I/O)."""
transform = transforms.Compose([
transforms.Resize(image_size),
transforms.CenterCrop(image_size),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
])
frame_paths = sorted(frames_dir.glob("*.png"))
if not frame_paths:
return np.zeros((0, 4, image_size // 8, image_size // 8), dtype=np.float32)
all_latents = []
dtype = torch.float16 if (use_fp16 and device.type == "cuda") else torch.float32
# Use ThreadPoolExecutor for parallel I/O (avoids Windows multiprocess overhead)
with ThreadPoolExecutor(max_workers=max(num_workers, 4)) as pool:
for i in range(0, len(frame_paths), batch_size):
batch_paths = frame_paths[i : i + batch_size]
# Load images in parallel threads
tensors = list(pool.map(
_load_and_transform,
[(p, transform) for p in batch_paths],
))
batch = torch.stack(tensors).to(device=device, dtype=dtype, non_blocking=True)
with torch.no_grad(), torch.amp.autocast(device_type=device.type, enabled=(device.type == "cuda")):
latent_dist = vae.encode(batch).latent_dist
latents = latent_dist.sample() * vae.config.scaling_factor
all_latents.append(latents.float().cpu().numpy())
return np.concatenate(all_latents, axis=0)
def main():
parser = argparse.ArgumentParser(description="Extract and cache features")
parser.add_argument("--input", type=str, required=True, help="Processed data directory")
parser.add_argument("--output", type=str, required=True, help="Cache output directory")
parser.add_argument("--image-size", type=int, default=256, help="Image size for VAE")
parser.add_argument("--batch-size", type=int, default=64, help="Batch size for VAE encoding (64 fits ~8GB VRAM with FP16)")
parser.add_argument("--device", type=str, default="cuda", help="Device")
parser.add_argument("--num-workers", type=int, default=0, help="DataLoader workers (0=main process, recommended on Windows)")
parser.add_argument("--no-fp16", action="store_true", help="Disable FP16 (use FP32)")
parser.add_argument(
"--skip-vae", action="store_true", help="Skip VAE latent extraction"
)
parser.add_argument(
"--skip-whisper", action="store_true", help="Skip Whisper feature extraction"
)
parser.add_argument(
"--skip-mel", action="store_true", help="Skip mel spectrogram extraction"
)
parser.add_argument(
"--vae-model",
type=str,
default="stable-diffusion-v1-5/stable-diffusion-v1-5",
help="VAE model ID",
)
parser.add_argument(
"--whisper-model",
type=str,
default="openai/whisper-tiny",
help="Whisper model ID",
)
args = parser.parse_args()
input_dir = Path(args.input)
output_dir = Path(args.output)
output_dir.mkdir(parents=True, exist_ok=True)
device = torch.device(args.device if torch.cuda.is_available() else "cpu")
use_fp16 = not args.no_fp16 and device.type == "cuda"
print(f"Using device: {device}")
print(f"FP16: {use_fp16} | Batch size: {args.batch_size} | Workers: {args.num_workers}")
if device.type == "cuda":
props = torch.cuda.get_device_properties(0)
print(f"GPU: {props.name} ({props.total_memory / 1024**3:.1f} GB VRAM)")
# Find all processed videos
video_dirs = [d for d in sorted(input_dir.iterdir()) if d.is_dir()]
if not video_dirs:
print(f"No processed video directories found in {input_dir}")
return
print(f"Found {len(video_dirs)} video directories")
# Load models as needed — FP16 on GPU for ~2× speed + half VRAM
vae = None
whisper_model = None
whisper_processor = None
dtype = torch.float16 if use_fp16 else torch.float32
if not args.skip_vae:
print("Loading VAE model...")
from diffusers import AutoencoderKL
vae = AutoencoderKL.from_pretrained(
args.vae_model, subfolder="vae", torch_dtype=dtype,
).to(device)
vae.eval()
vae.requires_grad_(False)
print("VAE loaded (FP16)." if use_fp16 else "VAE loaded (FP32).")
if not args.skip_whisper:
print("Loading Whisper model...")
from transformers import WhisperModel, WhisperProcessor
whisper_processor = WhisperProcessor.from_pretrained(args.whisper_model)
whisper_model = WhisperModel.from_pretrained(
args.whisper_model, torch_dtype=dtype,
).to(device)
whisper_model.eval()
whisper_model.requires_grad_(False)
print("Whisper loaded.")
# Process each video
t0 = time.time()
vae_done = whisper_done = mel_done = skipped = 0
for idx, video_dir in enumerate(tqdm(video_dirs, desc="Extracting features")):
video_name = video_dir.name
cache_dir = output_dir / video_name
cache_dir.mkdir(parents=True, exist_ok=True)
frames_dir = video_dir / "frames"
audio_path = video_dir / "audio.wav"
if not frames_dir.exists():
skipped += 1
continue
# 1. VAE Latents (GPU-accelerated, FP16, batched)
if not args.skip_vae:
latents_path = cache_dir / "vae_latents.npy"
if not latents_path.exists():
try:
latents = extract_vae_latents(
frames_dir, vae, device,
image_size=args.image_size,
batch_size=args.batch_size,
num_workers=args.num_workers,
use_fp16=use_fp16,
)
np.save(str(latents_path), latents)
vae_done += 1
except Exception as e:
print(f"VAE extraction failed for {video_name}: {e}")
# 2. Whisper Features (GPU-accelerated, FP16)
if not args.skip_whisper and audio_path.exists():
whisper_path = cache_dir / "whisper_features.npy"
if not whisper_path.exists():
try:
features = extract_whisper_features(
audio_path, whisper_model, whisper_processor, device,
use_fp16=use_fp16,
)
np.save(str(whisper_path), features)
whisper_done += 1
except Exception as e:
print(f"Whisper extraction failed for {video_name}: {e}")
# 3. Mel Spectrogram (CPU — fast, no GPU needed)
if not args.skip_mel and audio_path.exists():
mel_path = cache_dir / "mel_spectrogram.npy"
if not mel_path.exists():
try:
mel = extract_mel_spectrogram(audio_path)
np.save(str(mel_path), mel)
mel_done += 1
except Exception as e:
print(f"Mel extraction failed for {video_name}: {e}")
# Periodic GPU memory report
if device.type == "cuda" and (idx + 1) % 50 == 0:
mem = torch.cuda.memory_allocated() / 1024**3
peak = torch.cuda.max_memory_allocated() / 1024**3
print(f" [GPU] allocated: {mem:.1f} GB, peak: {peak:.1f} GB")
elapsed = time.time() - t0
# Clean up GPU memory
del vae, whisper_model
if torch.cuda.is_available():
torch.cuda.empty_cache()
print(f"\nFeature extraction complete in {elapsed / 60:.1f} minutes!")
print(f" VAE latents extracted: {vae_done}")
print(f" Whisper features extracted: {whisper_done}")
print(f" Mel spectrograms extracted: {mel_done}")
print(f" Skipped (no frames): {skipped}")
if __name__ == "__main__":
main()

Xet Storage Details

Size:
11.2 kB
·
Xet hash:
df1385f3a322a39f263d84c519bacf322ba0718e663504a5912ed0b12db26b10

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.