Buckets:
| """ | |
| Feature Extraction Script | |
| ========================== | |
| Pre-computes and caches expensive features: | |
| 1. VAE latent encoding of face frames | |
| 2. Whisper audio features | |
| 3. Mel spectrograms for SyncNet | |
| This avoids recomputing these on every training epoch. | |
| Usage: | |
| python scripts/extract_features.py --input data/processed --output data/cache | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| from pathlib import Path | |
| import librosa | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from PIL import Image | |
| from torchvision import transforms | |
| from tqdm import tqdm | |
| def extract_mel_spectrogram( | |
| audio_path: Path, | |
| sr: int = 16000, | |
| n_fft: int = 1024, | |
| hop_length: int = 256, | |
| n_mels: int = 80, | |
| ) -> np.ndarray: | |
| """Extract mel spectrogram from audio file.""" | |
| audio, _ = librosa.load(str(audio_path), sr=sr, mono=True) | |
| # Compute mel spectrogram | |
| mel = librosa.feature.melspectrogram( | |
| y=audio, | |
| sr=sr, | |
| n_fft=n_fft, | |
| hop_length=hop_length, | |
| n_mels=n_mels, | |
| fmin=50, | |
| fmax=7600, | |
| ) | |
| # Log scale | |
| mel = librosa.power_to_db(mel, ref=np.max) | |
| # Normalize to [-1, 1] | |
| mel = (mel - mel.min()) / (mel.max() - mel.min() + 1e-8) * 2 - 1 | |
| return mel.astype(np.float32) | |
| def extract_whisper_features( | |
| audio_path: Path, | |
| whisper_model, | |
| whisper_processor, | |
| device: torch.device, | |
| sr: int = 16000, | |
| chunk_duration: float = 30.0, | |
| use_fp16: bool = True, | |
| ) -> np.ndarray: | |
| """Extract Whisper encoder features from audio (GPU-accelerated, FP16).""" | |
| audio, _ = librosa.load(str(audio_path), sr=sr, mono=True) | |
| chunk_samples = int(chunk_duration * sr) | |
| all_features = [] | |
| dtype = torch.float16 if (use_fp16 and device.type == "cuda") else torch.float32 | |
| for start in range(0, len(audio), chunk_samples): | |
| chunk = audio[start : start + chunk_samples] | |
| if len(chunk) < sr: | |
| continue | |
| inputs = whisper_processor( | |
| chunk, sampling_rate=sr, return_tensors="pt", | |
| ) | |
| input_features = inputs.input_features.to(device=device, dtype=dtype) | |
| with torch.no_grad(), torch.amp.autocast(device_type=device.type, enabled=(device.type == "cuda")): | |
| encoder_output = whisper_model.encoder(input_features) | |
| features = encoder_output.last_hidden_state | |
| all_features.append(features.float().cpu().numpy()) | |
| if not all_features: | |
| return np.zeros((1, 1, 384), dtype=np.float32) | |
| return np.concatenate(all_features, axis=1) | |
| def _load_and_transform(args): | |
| """Load a single image and apply transform (for ThreadPool).""" | |
| path, transform = args | |
| img = Image.open(path).convert("RGB") | |
| return transform(img) | |
| def extract_vae_latents( | |
| frames_dir: Path, | |
| vae, | |
| device: torch.device, | |
| image_size: int = 256, | |
| batch_size: int = 64, | |
| num_workers: int = 4, | |
| use_fp16: bool = True, | |
| ) -> np.ndarray: | |
| """Encode face frames into VAE latent space (GPU-accelerated, FP16, threaded I/O).""" | |
| transform = transforms.Compose([ | |
| transforms.Resize(image_size), | |
| transforms.CenterCrop(image_size), | |
| transforms.ToTensor(), | |
| transforms.Normalize([0.5], [0.5]), | |
| ]) | |
| frame_paths = sorted(frames_dir.glob("*.png")) | |
| if not frame_paths: | |
| return np.zeros((0, 4, image_size // 8, image_size // 8), dtype=np.float32) | |
| all_latents = [] | |
| dtype = torch.float16 if (use_fp16 and device.type == "cuda") else torch.float32 | |
| # Use ThreadPoolExecutor for parallel I/O (avoids Windows multiprocess overhead) | |
| with ThreadPoolExecutor(max_workers=max(num_workers, 4)) as pool: | |
| for i in range(0, len(frame_paths), batch_size): | |
| batch_paths = frame_paths[i : i + batch_size] | |
| # Load images in parallel threads | |
| tensors = list(pool.map( | |
| _load_and_transform, | |
| [(p, transform) for p in batch_paths], | |
| )) | |
| batch = torch.stack(tensors).to(device=device, dtype=dtype, non_blocking=True) | |
| with torch.no_grad(), torch.amp.autocast(device_type=device.type, enabled=(device.type == "cuda")): | |
| latent_dist = vae.encode(batch).latent_dist | |
| latents = latent_dist.sample() * vae.config.scaling_factor | |
| all_latents.append(latents.float().cpu().numpy()) | |
| return np.concatenate(all_latents, axis=0) | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Extract and cache features") | |
| parser.add_argument("--input", type=str, required=True, help="Processed data directory") | |
| parser.add_argument("--output", type=str, required=True, help="Cache output directory") | |
| parser.add_argument("--image-size", type=int, default=256, help="Image size for VAE") | |
| parser.add_argument("--batch-size", type=int, default=64, help="Batch size for VAE encoding (64 fits ~8GB VRAM with FP16)") | |
| parser.add_argument("--device", type=str, default="cuda", help="Device") | |
| parser.add_argument("--num-workers", type=int, default=0, help="DataLoader workers (0=main process, recommended on Windows)") | |
| parser.add_argument("--no-fp16", action="store_true", help="Disable FP16 (use FP32)") | |
| parser.add_argument( | |
| "--skip-vae", action="store_true", help="Skip VAE latent extraction" | |
| ) | |
| parser.add_argument( | |
| "--skip-whisper", action="store_true", help="Skip Whisper feature extraction" | |
| ) | |
| parser.add_argument( | |
| "--skip-mel", action="store_true", help="Skip mel spectrogram extraction" | |
| ) | |
| parser.add_argument( | |
| "--vae-model", | |
| type=str, | |
| default="stable-diffusion-v1-5/stable-diffusion-v1-5", | |
| help="VAE model ID", | |
| ) | |
| parser.add_argument( | |
| "--whisper-model", | |
| type=str, | |
| default="openai/whisper-tiny", | |
| help="Whisper model ID", | |
| ) | |
| args = parser.parse_args() | |
| input_dir = Path(args.input) | |
| output_dir = Path(args.output) | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| device = torch.device(args.device if torch.cuda.is_available() else "cpu") | |
| use_fp16 = not args.no_fp16 and device.type == "cuda" | |
| print(f"Using device: {device}") | |
| print(f"FP16: {use_fp16} | Batch size: {args.batch_size} | Workers: {args.num_workers}") | |
| if device.type == "cuda": | |
| props = torch.cuda.get_device_properties(0) | |
| print(f"GPU: {props.name} ({props.total_memory / 1024**3:.1f} GB VRAM)") | |
| # Find all processed videos | |
| video_dirs = [d for d in sorted(input_dir.iterdir()) if d.is_dir()] | |
| if not video_dirs: | |
| print(f"No processed video directories found in {input_dir}") | |
| return | |
| print(f"Found {len(video_dirs)} video directories") | |
| # Load models as needed — FP16 on GPU for ~2× speed + half VRAM | |
| vae = None | |
| whisper_model = None | |
| whisper_processor = None | |
| dtype = torch.float16 if use_fp16 else torch.float32 | |
| if not args.skip_vae: | |
| print("Loading VAE model...") | |
| from diffusers import AutoencoderKL | |
| vae = AutoencoderKL.from_pretrained( | |
| args.vae_model, subfolder="vae", torch_dtype=dtype, | |
| ).to(device) | |
| vae.eval() | |
| vae.requires_grad_(False) | |
| print("VAE loaded (FP16)." if use_fp16 else "VAE loaded (FP32).") | |
| if not args.skip_whisper: | |
| print("Loading Whisper model...") | |
| from transformers import WhisperModel, WhisperProcessor | |
| whisper_processor = WhisperProcessor.from_pretrained(args.whisper_model) | |
| whisper_model = WhisperModel.from_pretrained( | |
| args.whisper_model, torch_dtype=dtype, | |
| ).to(device) | |
| whisper_model.eval() | |
| whisper_model.requires_grad_(False) | |
| print("Whisper loaded.") | |
| # Process each video | |
| t0 = time.time() | |
| vae_done = whisper_done = mel_done = skipped = 0 | |
| for idx, video_dir in enumerate(tqdm(video_dirs, desc="Extracting features")): | |
| video_name = video_dir.name | |
| cache_dir = output_dir / video_name | |
| cache_dir.mkdir(parents=True, exist_ok=True) | |
| frames_dir = video_dir / "frames" | |
| audio_path = video_dir / "audio.wav" | |
| if not frames_dir.exists(): | |
| skipped += 1 | |
| continue | |
| # 1. VAE Latents (GPU-accelerated, FP16, batched) | |
| if not args.skip_vae: | |
| latents_path = cache_dir / "vae_latents.npy" | |
| if not latents_path.exists(): | |
| try: | |
| latents = extract_vae_latents( | |
| frames_dir, vae, device, | |
| image_size=args.image_size, | |
| batch_size=args.batch_size, | |
| num_workers=args.num_workers, | |
| use_fp16=use_fp16, | |
| ) | |
| np.save(str(latents_path), latents) | |
| vae_done += 1 | |
| except Exception as e: | |
| print(f"VAE extraction failed for {video_name}: {e}") | |
| # 2. Whisper Features (GPU-accelerated, FP16) | |
| if not args.skip_whisper and audio_path.exists(): | |
| whisper_path = cache_dir / "whisper_features.npy" | |
| if not whisper_path.exists(): | |
| try: | |
| features = extract_whisper_features( | |
| audio_path, whisper_model, whisper_processor, device, | |
| use_fp16=use_fp16, | |
| ) | |
| np.save(str(whisper_path), features) | |
| whisper_done += 1 | |
| except Exception as e: | |
| print(f"Whisper extraction failed for {video_name}: {e}") | |
| # 3. Mel Spectrogram (CPU — fast, no GPU needed) | |
| if not args.skip_mel and audio_path.exists(): | |
| mel_path = cache_dir / "mel_spectrogram.npy" | |
| if not mel_path.exists(): | |
| try: | |
| mel = extract_mel_spectrogram(audio_path) | |
| np.save(str(mel_path), mel) | |
| mel_done += 1 | |
| except Exception as e: | |
| print(f"Mel extraction failed for {video_name}: {e}") | |
| # Periodic GPU memory report | |
| if device.type == "cuda" and (idx + 1) % 50 == 0: | |
| mem = torch.cuda.memory_allocated() / 1024**3 | |
| peak = torch.cuda.max_memory_allocated() / 1024**3 | |
| print(f" [GPU] allocated: {mem:.1f} GB, peak: {peak:.1f} GB") | |
| elapsed = time.time() - t0 | |
| # Clean up GPU memory | |
| del vae, whisper_model | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| print(f"\nFeature extraction complete in {elapsed / 60:.1f} minutes!") | |
| print(f" VAE latents extracted: {vae_done}") | |
| print(f" Whisper features extracted: {whisper_done}") | |
| print(f" Mel spectrograms extracted: {mel_done}") | |
| print(f" Skipped (no frames): {skipped}") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 11.2 kB
- Xet hash:
- df1385f3a322a39f263d84c519bacf322ba0718e663504a5912ed0b12db26b10
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.