Buckets:
| """ | |
| Main LipSync Model Training Script | |
| ===================================== | |
| Full training pipeline with: | |
| - Mixed precision (FP16/BF16) | |
| - Gradient checkpointing & accumulation | |
| - EMA (Exponential Moving Average) | |
| - TensorBoard logging | |
| - Checkpoint saving & resume | |
| - Validation with visual samples | |
| Usage: | |
| python -m src.training.train_model --config configs/train.yaml | |
| python -m src.training.train_model --config configs/test.yaml # Quick test | |
| """ | |
| import argparse | |
| import copy | |
| import gc | |
| import os | |
| import tempfile | |
| import time | |
| from pathlib import Path | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from omegaconf import OmegaConf | |
| from torch.amp import autocast, GradScaler | |
| from torch.utils.data import DataLoader, random_split | |
| from torch.utils.tensorboard import SummaryWriter | |
| from torchvision.utils import make_grid, save_image | |
| from tqdm import tqdm | |
| from src.data.dataset import LipSyncDataset | |
| from src.models.unet import LipSyncUNet | |
| from src.training.losses import CompositeLoss | |
| class EMA: | |
| """Exponential Moving Average for model parameters. | |
| Maintains a shadow copy of model weights that are updated as: | |
| shadow = decay * shadow + (1 - decay) * current | |
| EMA weights typically produce smoother and better results at inference. | |
| """ | |
| def __init__(self, model: nn.Module, decay: float = 0.9999, update_every: int = 10): | |
| self.decay = decay | |
| self.update_every = update_every | |
| self.step = 0 | |
| self.shadow = {} | |
| self.backup = {} | |
| for name, param in model.named_parameters(): | |
| if param.requires_grad: | |
| self.shadow[name] = param.data.clone() | |
| def update(self, model: nn.Module): | |
| self.step += 1 | |
| if self.step % self.update_every != 0: | |
| return | |
| for name, param in model.named_parameters(): | |
| if param.requires_grad and name in self.shadow: | |
| self.shadow[name].lerp_(param.data, 1 - self.decay) | |
| def apply(self, model: nn.Module): | |
| """Apply EMA weights to model (for evaluation).""" | |
| for name, param in model.named_parameters(): | |
| if param.requires_grad and name in self.shadow: | |
| self.backup[name] = param.data.clone() | |
| param.data.copy_(self.shadow[name]) | |
| def restore(self, model: nn.Module): | |
| """Restore original weights after evaluation.""" | |
| for name, param in model.named_parameters(): | |
| if param.requires_grad and name in self.backup: | |
| param.data.copy_(self.backup[name]) | |
| self.backup = {} | |
| def state_dict(self): | |
| return {"shadow": self.shadow, "step": self.step, "decay": self.decay} | |
| def load_state_dict(self, state_dict): | |
| self.shadow = state_dict["shadow"] | |
| self.step = state_dict["step"] | |
| self.decay = state_dict["decay"] | |
| def create_lr_scheduler(optimizer, config, total_steps): | |
| """Create learning rate scheduler with warmup.""" | |
| warmup_steps = config.training.scheduler.warmup_steps | |
| min_lr = config.training.scheduler.min_lr | |
| base_lr = config.training.learning_rate | |
| def lr_lambda(step): | |
| if step < warmup_steps: | |
| return step / max(1, warmup_steps) | |
| progress = (step - warmup_steps) / max(1, total_steps - warmup_steps) | |
| return max(min_lr / base_lr, 0.5 * (1 + torch.cos(torch.tensor(progress * 3.14159)).item())) | |
| return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda) | |
| def save_checkpoint( | |
| path: Path, | |
| epoch: int, | |
| global_step: int, | |
| model: nn.Module, | |
| optimizer, | |
| scheduler, | |
| scaler, | |
| ema, | |
| best_val_loss: float, | |
| config, | |
| patience_counter: int = 0, | |
| ): | |
| """Save training checkpoint.""" | |
| state = { | |
| "epoch": epoch, | |
| "global_step": global_step, | |
| "model_state_dict": model.state_dict(), | |
| "optimizer_state_dict": optimizer.state_dict(), | |
| "scheduler_state_dict": scheduler.state_dict(), | |
| "scaler_state_dict": scaler.state_dict() if scaler.is_enabled() else None, | |
| "best_val_loss": best_val_loss, | |
| "patience_counter": patience_counter, | |
| "config": OmegaConf.to_container(config), | |
| } | |
| if ema is not None: | |
| state["ema_state_dict"] = ema.state_dict() | |
| # Atomic save: write to temp file then rename to prevent corruption | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp") | |
| try: | |
| os.close(fd) | |
| torch.save(state, tmp_path) | |
| os.replace(tmp_path, str(path)) | |
| except BaseException: | |
| if os.path.exists(tmp_path): | |
| os.remove(tmp_path) | |
| raise | |
| def load_checkpoint(path: Path, model, optimizer, scheduler, scaler, ema, device): | |
| """Load training checkpoint.""" | |
| ckpt = torch.load(path, map_location=device, weights_only=False) | |
| model.load_state_dict(ckpt["model_state_dict"]) | |
| optimizer.load_state_dict(ckpt["optimizer_state_dict"]) | |
| scheduler.load_state_dict(ckpt["scheduler_state_dict"]) | |
| if scaler.is_enabled() and ckpt.get("scaler_state_dict"): | |
| scaler.load_state_dict(ckpt["scaler_state_dict"]) | |
| if ema is not None and "ema_state_dict" in ckpt: | |
| ema.load_state_dict(ckpt["ema_state_dict"]) | |
| return ckpt["epoch"], ckpt["global_step"], ckpt.get("best_val_loss", float("inf")), ckpt.get("patience_counter", 0) | |
| def validate( | |
| model: nn.Module, | |
| val_loader: DataLoader, | |
| loss_fn: CompositeLoss, | |
| device: torch.device, | |
| amp_dtype: torch.dtype, | |
| use_amp: bool, | |
| writer: SummaryWriter, | |
| global_step: int, | |
| num_samples: int = 4, | |
| vae=None, | |
| ): | |
| """Run validation and log visual samples.""" | |
| model.eval() | |
| total_loss = 0 | |
| n_batches = 0 | |
| sample_preds = [] | |
| sample_targets = [] | |
| sample_masked = [] | |
| with torch.no_grad(): | |
| for batch in val_loader: | |
| target_latent = batch["target_latent"].to(device) | |
| ref_latent = batch["reference_latent"].to(device) | |
| audio_features = batch["audio_features"].to(device) | |
| B, C, H, W = target_latent.shape | |
| # Create mask | |
| mask = torch.zeros(B, 1, H, W, device=device) | |
| mask[:, :, H // 2 :, :] = 1.0 | |
| masked_latent = target_latent * (1 - mask) | |
| with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp): | |
| pred_latent = model( | |
| masked_latent=masked_latent, | |
| reference_latent=ref_latent, | |
| mask=mask, | |
| audio_features=audio_features, | |
| ) | |
| losses = loss_fn(pred_latent, target_latent, mask) | |
| total_loss += losses["total"].item() | |
| n_batches += 1 | |
| # Collect samples for visualization | |
| if len(sample_preds) < num_samples: | |
| sample_preds.append(pred_latent[:1]) | |
| sample_targets.append(target_latent[:1]) | |
| sample_masked.append(masked_latent[:1]) | |
| avg_loss = total_loss / max(1, n_batches) | |
| writer.add_scalar("val/loss", avg_loss, global_step) | |
| # Decode and visualize samples if VAE is available | |
| if vae is not None and sample_preds: | |
| try: | |
| pred_latents = torch.cat(sample_preds[:num_samples], dim=0) | |
| target_latents = torch.cat(sample_targets[:num_samples], dim=0) | |
| masked_latents = torch.cat(sample_masked[:num_samples], dim=0) | |
| scale = vae.config.scaling_factor | |
| with torch.no_grad(): | |
| pred_imgs = vae.decode(pred_latents / scale).sample | |
| target_imgs = vae.decode(target_latents / scale).sample | |
| masked_imgs = vae.decode(masked_latents / scale).sample | |
| # Create comparison grid: masked | predicted | ground truth | |
| comparison = torch.cat([masked_imgs, pred_imgs, target_imgs], dim=0) | |
| grid = make_grid( | |
| comparison.clamp(-1, 1) * 0.5 + 0.5, # Scale to [0, 1] | |
| nrow=len(sample_preds), | |
| padding=2, | |
| ) | |
| writer.add_image("val/comparison", grid, global_step) | |
| except Exception as e: | |
| print(f"Warning: Could not generate validation images: {e}") | |
| model.train() | |
| return avg_loss | |
| def train(config): | |
| """Main training loop.""" | |
| device = torch.device(config.hardware.device if torch.cuda.is_available() else "cpu") | |
| print(f"Device: {device}") | |
| if torch.cuda.is_available(): | |
| print(f"GPU: {torch.cuda.get_device_name()}") | |
| print(f"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB") | |
| # === Dataset === | |
| dataset = LipSyncDataset( | |
| processed_dir=config.data.processed_dir, | |
| cache_dir=config.data.cache_dir, | |
| image_size=config.data.image_size, | |
| fps=config.data.fps, | |
| mel_step_size=config.data.mel_step_size, | |
| num_audio_frames=config.data.num_audio_frames, | |
| sample_rate=config.data.sample_rate, | |
| ) | |
| if len(dataset) == 0: | |
| print("ERROR: Dataset is empty. Run preprocessing and feature extraction first.") | |
| return | |
| # Train/val split | |
| val_size = max(1, int(len(dataset) * 0.05)) | |
| train_size = len(dataset) - val_size | |
| train_dataset, val_dataset = random_split(dataset, [train_size, val_size]) | |
| train_loader = DataLoader( | |
| train_dataset, | |
| batch_size=config.training.batch_size, | |
| shuffle=True, | |
| num_workers=config.data.num_workers, | |
| pin_memory=config.data.pin_memory, | |
| drop_last=True, | |
| ) | |
| val_loader = DataLoader( | |
| val_dataset, | |
| batch_size=config.training.batch_size, | |
| shuffle=False, | |
| num_workers=config.data.num_workers, | |
| pin_memory=config.data.pin_memory, | |
| ) | |
| print(f"Train: {len(train_dataset)}, Val: {len(val_dataset)}") | |
| # === Model (UNet only - frozen components loaded separately) === | |
| unet_cfg = OmegaConf.to_container(config.model.unet) | |
| model = LipSyncUNet(**unet_cfg).to(device) | |
| param_info = model.get_param_count() | |
| print(f"UNet parameters: {param_info['trainable_M']:.2f}M trainable") | |
| # === Load frozen VAE (for validation visualization) === | |
| vae = None | |
| try: | |
| from diffusers import AutoencoderKL | |
| vae = AutoencoderKL.from_pretrained( | |
| config.model.vae_model_id, subfolder=config.model.vae_subfolder | |
| ).to(device) | |
| vae.eval() | |
| vae.requires_grad_(False) | |
| print("VAE loaded for loss computation + visualization") | |
| except Exception as e: | |
| print(f"VAE not loaded (perceptual/sync/GAN losses disabled): {e}") | |
| # === Loss function === | |
| syncnet = None | |
| syncnet_path = config.model.syncnet.get("checkpoint") if hasattr(config.model, "syncnet") else None | |
| if syncnet_path and Path(syncnet_path).exists(): | |
| from src.models.syncnet import SyncNet | |
| syncnet = SyncNet().to(device) | |
| ckpt = torch.load(syncnet_path, map_location=device, weights_only=True) | |
| syncnet.load_state_dict(ckpt["model_state_dict"]) | |
| syncnet.eval() | |
| syncnet.requires_grad_(False) | |
| print(f"SyncNet loaded from {syncnet_path}") | |
| loss_fn = CompositeLoss( | |
| weight_l1=config.training.loss_weights.l1, | |
| weight_perceptual=config.training.loss_weights.perceptual, | |
| weight_sync=config.training.loss_weights.sync, | |
| weight_gan=config.training.loss_weights.gan, | |
| syncnet=syncnet, | |
| ).to(device) | |
| # === Optimizer === | |
| optimizer = torch.optim.AdamW( | |
| model.parameters(), | |
| lr=config.training.learning_rate, | |
| weight_decay=config.training.weight_decay, | |
| betas=tuple(config.training.betas), | |
| ) | |
| # Scheduler | |
| steps_per_epoch = len(train_loader) // config.training.gradient_accumulation_steps | |
| total_steps = steps_per_epoch * config.training.max_epochs | |
| scheduler = create_lr_scheduler(optimizer, config, total_steps) | |
| # Mixed precision | |
| use_amp = config.training.mixed_precision in ("fp16", "bf16") | |
| scaler = GradScaler(enabled=use_amp and config.training.mixed_precision == "fp16") | |
| amp_dtype = torch.bfloat16 if config.training.mixed_precision == "bf16" else torch.float16 | |
| # EMA | |
| ema = None | |
| if config.training.ema.get("enabled", False): | |
| ema = EMA( | |
| model, | |
| decay=config.training.ema.decay, | |
| update_every=config.training.ema.update_every, | |
| ) | |
| print(f"EMA enabled (decay={config.training.ema.decay})") | |
| # Logging | |
| log_dir = Path(config.training.log_dir) | |
| log_dir.mkdir(parents=True, exist_ok=True) | |
| writer = SummaryWriter(log_dir=str(log_dir)) | |
| checkpoint_dir = Path(config.training.checkpoint_dir) | |
| checkpoint_dir.mkdir(parents=True, exist_ok=True) | |
| # Resume | |
| start_epoch = 0 | |
| global_step = 0 | |
| best_val_loss = float("inf") | |
| patience_counter = 0 | |
| patience = config.training.get("early_stopping_patience", 0) # 0 = disabled | |
| if config.training.resume_from and Path(config.training.resume_from).exists(): | |
| print(f"Resuming from {config.training.resume_from}") | |
| start_epoch, global_step, best_val_loss, patience_counter = load_checkpoint( | |
| Path(config.training.resume_from), model, optimizer, scheduler, scaler, ema, device | |
| ) | |
| print(f"Resumed at epoch {start_epoch}, step {global_step}, patience={patience_counter}/{patience}") | |
| # === Training Loop === | |
| accum_steps = config.training.gradient_accumulation_steps | |
| for epoch in range(start_epoch, config.training.max_epochs): | |
| model.train() | |
| epoch_loss = 0 | |
| epoch_losses = {} | |
| n_batches = 0 | |
| pbar = tqdm( | |
| train_loader, | |
| desc=f"Epoch {epoch+1}/{config.training.max_epochs}", | |
| ) | |
| optimizer.zero_grad() | |
| for batch_idx, batch in enumerate(pbar): | |
| target_latent = batch["target_latent"].to(device) | |
| ref_latent = batch["reference_latent"].to(device) | |
| audio_features = batch["audio_features"].to(device) | |
| target_image = batch["target_image"].to(device) | |
| mel_segment = batch["mel_segment"].to(device) | |
| lip_frames = batch["lip_frames"].to(device) | |
| B, C, H, W = target_latent.shape | |
| # Create mask | |
| mask = torch.zeros(B, 1, H, W, device=device) | |
| mask[:, :, H // 2 :, :] = 1.0 | |
| masked_latent = target_latent * (1 - mask) | |
| # Forward pass with mixed precision | |
| with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp): | |
| pred_latent = model( | |
| masked_latent=masked_latent, | |
| reference_latent=ref_latent, | |
| mask=mask, | |
| audio_features=audio_features, | |
| ) | |
| # Decode predicted latents to pixel space for perceptual/sync/GAN losses | |
| pred_image = None | |
| pred_lip_frames = None | |
| if vae is not None: | |
| with torch.no_grad(): | |
| scale = vae.config.scaling_factor | |
| pred_image = vae.decode(pred_latent / scale).sample | |
| # Resize to match target_image if resolutions differ | |
| target_h = target_image.shape[2] | |
| if pred_image.shape[2] != target_h: | |
| pred_image = F.interpolate( | |
| pred_image, size=(target_h, target_h), | |
| mode="bilinear", align_corners=False, | |
| ) | |
| # Crop lip region from predicted image for SyncNet loss | |
| img_h = pred_image.shape[2] | |
| lip_crop = pred_image[:, :, img_h // 2:, :] # Lower half | |
| lip_crop = F.interpolate(lip_crop, size=(96, 96), mode="bilinear", align_corners=False) | |
| # SyncNet expects (B, T, C, H, W) — use single predicted frame as center | |
| T = lip_frames.shape[1] | |
| pred_lip_frames = lip_crop.unsqueeze(1).expand(-1, T, -1, -1, -1) | |
| # Compute losses (perceptual/sync/GAN activate when images are available) | |
| losses = loss_fn( | |
| pred_latent=pred_latent, | |
| target_latent=target_latent, | |
| mask=mask, | |
| pred_image=pred_image, | |
| target_image=target_image, | |
| lip_frames=pred_lip_frames, | |
| mel_segment=mel_segment, | |
| ) | |
| loss = losses["total"] / accum_steps | |
| # Backward | |
| if scaler.is_enabled(): | |
| scaler.scale(loss).backward() | |
| else: | |
| loss.backward() | |
| # Accumulate gradients | |
| if (batch_idx + 1) % accum_steps == 0: | |
| if scaler.is_enabled(): | |
| scaler.unscale_(optimizer) | |
| nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| if scaler.is_enabled(): | |
| scaler.step(optimizer) | |
| scaler.update() | |
| else: | |
| optimizer.step() | |
| optimizer.zero_grad() | |
| scheduler.step() | |
| if ema is not None: | |
| ema.update(model) | |
| global_step += 1 | |
| # Track losses | |
| epoch_loss += losses["total"].item() | |
| for k, v in losses.items(): | |
| if k != "total": | |
| epoch_losses[k] = epoch_losses.get(k, 0) + v.item() | |
| n_batches += 1 | |
| # Progress bar | |
| pbar.set_postfix( | |
| loss=f"{losses['total'].item():.4f}", | |
| lr=f"{scheduler.get_last_lr()[0]:.2e}", | |
| ) | |
| # Logging | |
| if global_step > 0 and global_step % config.training.log_every_n_steps == 0: | |
| writer.add_scalar("train/loss", losses["total"].item(), global_step) | |
| writer.add_scalar("train/lr", scheduler.get_last_lr()[0], global_step) | |
| for k, v in losses.items(): | |
| if k != "total": | |
| writer.add_scalar(f"train/{k}", v.item(), global_step) | |
| avg_loss = epoch_loss / max(1, n_batches) | |
| print(f"Epoch {epoch+1} - Loss: {avg_loss:.4f}") | |
| for k, v in epoch_losses.items(): | |
| print(f" {k}: {v / max(1, n_batches):.4f}") | |
| # === Validation === | |
| if (epoch + 1) % config.training.val_every_n_epochs == 0: | |
| if ema is not None: | |
| ema.apply(model) | |
| val_loss = validate( | |
| model=model, | |
| val_loader=val_loader, | |
| loss_fn=loss_fn, | |
| device=device, | |
| amp_dtype=amp_dtype, | |
| use_amp=use_amp, | |
| writer=writer, | |
| global_step=global_step, | |
| num_samples=config.training.num_val_samples, | |
| vae=vae, | |
| ) | |
| print(f" Val Loss: {val_loss:.4f}") | |
| if ema is not None: | |
| ema.restore(model) | |
| # Save best | |
| if val_loss < best_val_loss: | |
| best_val_loss = val_loss | |
| patience_counter = 0 | |
| save_checkpoint( | |
| checkpoint_dir / "best_model.pt", | |
| epoch, global_step, model, optimizer, scheduler, scaler, ema, | |
| best_val_loss, config, patience_counter, | |
| ) | |
| print(f" Saved best model (val_loss={val_loss:.4f})") | |
| else: | |
| patience_counter += 1 | |
| if patience > 0: | |
| print(f" No improvement ({patience_counter}/{patience})") | |
| if patience_counter >= patience: | |
| print(f" Early stopping triggered after {patience_counter} validations without improvement.") | |
| break | |
| # Periodic checkpoint | |
| if (epoch + 1) % config.training.save_every_n_epochs == 0: | |
| save_checkpoint( | |
| checkpoint_dir / f"checkpoint_epoch_{epoch+1}.pt", | |
| epoch, global_step, model, optimizer, scheduler, scaler, ema, | |
| best_val_loss, config, patience_counter, | |
| ) | |
| # Keep only top-k checkpoints | |
| ckpts = sorted( | |
| checkpoint_dir.glob("checkpoint_epoch_*.pt"), | |
| key=lambda p: p.stat().st_mtime, | |
| reverse=True, | |
| ) | |
| for old_ckpt in ckpts[config.training.save_top_k:]: | |
| old_ckpt.unlink() | |
| # Save final model | |
| save_checkpoint( | |
| checkpoint_dir / "final_model.pt", | |
| config.training.max_epochs, global_step, model, optimizer, scheduler, scaler, ema, | |
| best_val_loss, config, patience_counter, | |
| ) | |
| writer.close() | |
| print(f"\nTraining complete! Best val loss: {best_val_loss:.4f}") | |
| print(f"Checkpoints saved to: {checkpoint_dir}") | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Train LipSync Model") | |
| parser.add_argument("--config", type=str, default="configs/train.yaml") | |
| args = parser.parse_args() | |
| config = OmegaConf.load(args.config) | |
| train(config) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 22 kB
- Xet hash:
- a8526e6f712cf3b21948d5ff04e00ff4fa63f34b11910ecaa773891c4dec0c49
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.