Buckets:
| """ | |
| Main LipSync Model Distributed Training Script | |
| =================================================== | |
| Multi-GPU distributed training for the main LipSync model using PyTorch DDP. | |
| Features: | |
| - DistributedDataParallel across multiple GPUs | |
| - Gradient accumulation with proper DDP sync control | |
| - EMA (Exponential Moving Average) support | |
| - Resume from single-GPU or multi-GPU checkpoints | |
| - Frozen VAE + Whisper stay on each GPU (small enough to replicate) | |
| - Only UNet gradients are synchronized across GPUs | |
| - Validation with visual samples (rank 0 only) | |
| Usage (single-node, 4 GPUs): | |
| torchrun --nproc_per_node=4 \ | |
| -m src.training.train_model_distributed \ | |
| --config configs/train_distributed.yaml \ | |
| --resume checkpoints/best_model.pt | |
| Usage (multi-node, 2 nodes × 4 GPUs): | |
| # Node 0 (master): | |
| torchrun --nproc_per_node=4 --nnodes=2 --node_rank=0 \ | |
| --master_addr=<IP> --master_port=29500 \ | |
| -m src.training.train_model_distributed \ | |
| --config configs/train_distributed.yaml \ | |
| --resume checkpoints/best_model.pt | |
| # Node 1: | |
| torchrun --nproc_per_node=4 --nnodes=2 --node_rank=1 \ | |
| --master_addr=<IP> --master_port=29500 \ | |
| -m src.training.train_model_distributed \ | |
| --config configs/train_distributed.yaml \ | |
| --resume checkpoints/best_model.pt | |
| """ | |
| import argparse | |
| import copy | |
| import gc | |
| import os | |
| import tempfile | |
| import time | |
| from contextlib import nullcontext | |
| from pathlib import Path | |
| import torch | |
| import torch.distributed as dist | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from omegaconf import OmegaConf | |
| from torch.amp import autocast, GradScaler | |
| from torch.nn.parallel import DistributedDataParallel as DDP | |
| from torch.utils.data import DataLoader, random_split | |
| from torch.utils.data.distributed import DistributedSampler | |
| from torch.utils.tensorboard import SummaryWriter | |
| from torchvision.utils import make_grid | |
| from tqdm import tqdm | |
| from src.data.dataset import LipSyncDataset | |
| from src.models.unet import LipSyncUNet | |
| from src.training.losses import CompositeLoss | |
| from src.training.notify import send_training_complete | |
| # ================================================================ | |
| # EMA — Exponential Moving Average | |
| # ================================================================ | |
| class EMA: | |
| """Exponential Moving Average for model parameters. | |
| Maintains a shadow copy of model weights that are updated as: | |
| shadow = decay * shadow + (1 - decay) * current | |
| EMA weights typically produce smoother and better results at inference. | |
| Works with DDP by operating on the unwrapped model. | |
| """ | |
| def __init__(self, model: nn.Module, decay: float = 0.9999, update_every: int = 10): | |
| self.decay = decay | |
| self.update_every = update_every | |
| self.step = 0 | |
| self.shadow = {} | |
| self.backup = {} | |
| for name, param in model.named_parameters(): | |
| if param.requires_grad: | |
| self.shadow[name] = param.data.clone() | |
| def update(self, model: nn.Module): | |
| self.step += 1 | |
| if self.step % self.update_every != 0: | |
| return | |
| for name, param in model.named_parameters(): | |
| if param.requires_grad and name in self.shadow: | |
| self.shadow[name].lerp_(param.data, 1 - self.decay) | |
| def apply(self, model: nn.Module): | |
| """Apply EMA weights to model (for evaluation).""" | |
| for name, param in model.named_parameters(): | |
| if param.requires_grad and name in self.shadow: | |
| self.backup[name] = param.data.clone() | |
| param.data.copy_(self.shadow[name]) | |
| def restore(self, model: nn.Module): | |
| """Restore original weights after evaluation.""" | |
| for name, param in model.named_parameters(): | |
| if param.requires_grad and name in self.backup: | |
| param.data.copy_(self.backup[name]) | |
| self.backup = {} | |
| def state_dict(self): | |
| return {"shadow": self.shadow, "step": self.step, "decay": self.decay} | |
| def load_state_dict(self, state_dict): | |
| self.shadow = state_dict["shadow"] | |
| self.step = state_dict["step"] | |
| self.decay = state_dict["decay"] | |
| # ================================================================ | |
| # Utility functions | |
| # ================================================================ | |
| def setup_distributed(): | |
| """Initialize distributed process group.""" | |
| dist.init_process_group(backend="nccl") | |
| local_rank = int(os.environ["LOCAL_RANK"]) | |
| torch.cuda.set_device(local_rank) | |
| return local_rank | |
| def cleanup_distributed(): | |
| """Clean up distributed process group.""" | |
| if dist.is_initialized(): | |
| dist.destroy_process_group() | |
| def print_rank0(msg, rank=0): | |
| """Print only on rank 0.""" | |
| if rank == 0: | |
| print(msg) | |
| def atomic_save(state, path): | |
| """Save checkpoint atomically to prevent corruption from pod interrupts. | |
| Writes to a temporary file first, then renames. This ensures the | |
| checkpoint file is either fully written or not present at all. | |
| """ | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| # Write to temp file in same directory (same filesystem for atomic rename) | |
| fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp") | |
| try: | |
| os.close(fd) | |
| torch.save(state, tmp_path) | |
| # Atomic rename (on POSIX; on Windows this may not be atomic but is still safer) | |
| os.replace(tmp_path, str(path)) | |
| except BaseException: | |
| # Clean up temp file on any failure (including KeyboardInterrupt) | |
| if os.path.exists(tmp_path): | |
| os.remove(tmp_path) | |
| raise | |
| def verify_checkpoint(ckpt, resume_path, rank): | |
| """Verify checkpoint integrity and log diagnostic info.""" | |
| required_keys = ["epoch", "global_step", "model_state_dict"] | |
| missing = [k for k in required_keys if k not in ckpt] | |
| if missing: | |
| print_rank0(f"WARNING: Checkpoint missing keys: {missing}", rank) | |
| return False | |
| state_dict = ckpt["model_state_dict"] | |
| num_params = len(state_dict) | |
| total_elements = sum(v.numel() for v in state_dict.values()) | |
| # Quick NaN/Inf check on a sample of tensors | |
| has_nan = False | |
| has_inf = False | |
| sample_vals = [] | |
| for i, (name, tensor) in enumerate(state_dict.items()): | |
| if torch.isnan(tensor).any(): | |
| has_nan = True | |
| print_rank0(f" WARNING: NaN detected in checkpoint param: {name}", rank) | |
| if torch.isinf(tensor).any(): | |
| has_inf = True | |
| print_rank0(f" WARNING: Inf detected in checkpoint param: {name}", rank) | |
| if i < 3: # Log first 3 param stats | |
| sample_vals.append(f"{name}: mean={tensor.float().mean():.6f}, std={tensor.float().std():.6f}") | |
| print_rank0(f" Checkpoint: {num_params} params, {total_elements:,} elements", rank) | |
| for sv in sample_vals: | |
| print_rank0(f" {sv}", rank) | |
| print_rank0( | |
| f" Saved at: epoch={ckpt.get('epoch', '?')}, step={ckpt.get('global_step', '?')}, " | |
| f"best_val_loss={ckpt.get('best_val_loss', '?')}", | |
| rank, | |
| ) | |
| if ckpt.get("world_size"): | |
| print_rank0(f" Original world_size: {ckpt['world_size']}", rank) | |
| if has_nan or has_inf: | |
| print_rank0(" ERROR: Checkpoint contains NaN/Inf values — likely corrupted!", rank) | |
| return False | |
| return True | |
| def create_lr_scheduler(optimizer, config, total_steps, base_lr): | |
| """Create learning rate scheduler with warmup.""" | |
| warmup_steps = config.training.scheduler.warmup_steps | |
| min_lr = config.training.scheduler.min_lr | |
| def lr_lambda(step): | |
| if step < warmup_steps: | |
| return step / max(1, warmup_steps) | |
| progress = (step - warmup_steps) / max(1, total_steps - warmup_steps) | |
| return max( | |
| min_lr / base_lr, | |
| 0.5 * (1 + torch.cos(torch.tensor(progress * 3.14159)).item()), | |
| ) | |
| return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda) | |
| def validate_distributed( | |
| model: nn.Module, | |
| val_loader: DataLoader, | |
| loss_fn: CompositeLoss, | |
| device: torch.device, | |
| amp_dtype: torch.dtype, | |
| use_amp: bool, | |
| writer, | |
| global_step: int, | |
| rank: int, | |
| num_samples: int = 4, | |
| vae=None, | |
| ): | |
| """Run validation across all GPUs, aggregate results, log on rank 0.""" | |
| model.eval() | |
| total_loss = 0 | |
| n_batches = 0 | |
| sample_preds = [] | |
| sample_targets = [] | |
| sample_masked = [] | |
| with torch.no_grad(): | |
| for batch in val_loader: | |
| target_latent = batch["target_latent"].to(device, non_blocking=True) | |
| ref_latent = batch["reference_latent"].to(device, non_blocking=True) | |
| audio_features = batch["audio_features"].to(device, non_blocking=True) | |
| B, C, H, W = target_latent.shape | |
| mask = torch.zeros(B, 1, H, W, device=device) | |
| mask[:, :, H // 2:, :] = 1.0 | |
| masked_latent = target_latent * (1 - mask) | |
| with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp): | |
| # Access the unwrapped model for inference | |
| unet = model.module if hasattr(model, "module") else model | |
| pred_latent = unet( | |
| masked_latent=masked_latent, | |
| reference_latent=ref_latent, | |
| mask=mask, | |
| audio_features=audio_features, | |
| ) | |
| losses = loss_fn(pred_latent, target_latent, mask) | |
| total_loss += losses["total"].item() | |
| n_batches += 1 | |
| # Collect samples for visualization (rank 0 only) | |
| if rank == 0 and len(sample_preds) < num_samples: | |
| sample_preds.append(pred_latent[:1]) | |
| sample_targets.append(target_latent[:1]) | |
| sample_masked.append(masked_latent[:1]) | |
| # Aggregate validation loss across all GPUs | |
| loss_tensor = torch.tensor([total_loss, n_batches], device=device, dtype=torch.float64) | |
| dist.all_reduce(loss_tensor, op=dist.ReduceOp.SUM) | |
| avg_loss = loss_tensor[0].item() / max(1, loss_tensor[1].item()) | |
| if rank == 0 and writer is not None: | |
| writer.add_scalar("val/loss", avg_loss, global_step) | |
| # Decode and visualize samples if VAE available | |
| if vae is not None and sample_preds: | |
| try: | |
| pred_latents = torch.cat(sample_preds[:num_samples], dim=0) | |
| target_latents = torch.cat(sample_targets[:num_samples], dim=0) | |
| masked_latents = torch.cat(sample_masked[:num_samples], dim=0) | |
| scale = vae.config.scaling_factor | |
| with torch.no_grad(): | |
| pred_imgs = vae.decode(pred_latents / scale).sample | |
| target_imgs = vae.decode(target_latents / scale).sample | |
| masked_imgs = vae.decode(masked_latents / scale).sample | |
| comparison = torch.cat([masked_imgs, pred_imgs, target_imgs], dim=0) | |
| grid = make_grid( | |
| comparison.clamp(-1, 1) * 0.5 + 0.5, | |
| nrow=len(sample_preds), | |
| padding=2, | |
| ) | |
| writer.add_image("val/comparison", grid, global_step) | |
| except Exception as e: | |
| print(f"Warning: Could not generate validation images: {e}") | |
| model.train() | |
| return avg_loss | |
| # ================================================================ | |
| # Main Training Function | |
| # ================================================================ | |
| def train_distributed(config, resume_path=None): | |
| """Main distributed training loop for the LipSync model.""" | |
| # === Distributed Setup === | |
| local_rank = setup_distributed() | |
| rank = dist.get_rank() | |
| world_size = dist.get_world_size() | |
| device = torch.device(f"cuda:{local_rank}") | |
| print_rank0(f"Distributed training initialized: {world_size} GPUs", rank) | |
| print_rank0(f" Backend: {dist.get_backend()}", rank) | |
| print(f" Rank {rank}/{world_size} → GPU {local_rank} ({torch.cuda.get_device_name(local_rank)})") | |
| # === Dataset === | |
| dataset = LipSyncDataset( | |
| processed_dir=config.data.processed_dir, | |
| cache_dir=config.data.cache_dir, | |
| image_size=config.data.image_size, | |
| fps=config.data.fps, | |
| mel_step_size=config.data.mel_step_size, | |
| num_audio_frames=config.data.num_audio_frames, | |
| sample_rate=config.data.sample_rate, | |
| ) | |
| if len(dataset) == 0: | |
| print_rank0("ERROR: Dataset is empty. Run preprocessing and feature extraction first.", rank) | |
| cleanup_distributed() | |
| return | |
| # Train/val split — fixed seed for all ranks | |
| val_size = max(1, int(len(dataset) * 0.05)) | |
| train_size = len(dataset) - val_size | |
| split_generator = torch.Generator().manual_seed(42) | |
| train_dataset, val_dataset = random_split( | |
| dataset, [train_size, val_size], generator=split_generator | |
| ) | |
| # === Distributed Samplers === | |
| train_sampler = DistributedSampler( | |
| train_dataset, num_replicas=world_size, rank=rank, shuffle=True | |
| ) | |
| val_sampler = DistributedSampler( | |
| val_dataset, num_replicas=world_size, rank=rank, shuffle=False | |
| ) | |
| train_loader = DataLoader( | |
| train_dataset, | |
| batch_size=config.training.batch_size, # per-GPU batch size | |
| shuffle=False, | |
| sampler=train_sampler, | |
| num_workers=config.data.num_workers, | |
| pin_memory=config.data.pin_memory, | |
| drop_last=True, | |
| ) | |
| val_loader = DataLoader( | |
| val_dataset, | |
| batch_size=config.training.batch_size, | |
| shuffle=False, | |
| sampler=val_sampler, | |
| num_workers=config.data.num_workers, | |
| pin_memory=config.data.pin_memory, | |
| ) | |
| effective_batch_size = config.training.batch_size * world_size * config.training.gradient_accumulation_steps | |
| print_rank0( | |
| f"Train: {len(train_dataset)}, Val: {len(val_dataset)}\n" | |
| f"Per-GPU batch: {config.training.batch_size}, " | |
| f"Accum steps: {config.training.gradient_accumulation_steps}, " | |
| f"Effective batch: {effective_batch_size}", | |
| rank, | |
| ) | |
| # === Model (UNet only — trainable) === | |
| unet_cfg = OmegaConf.to_container(config.model.unet) | |
| model = LipSyncUNet(**unet_cfg).to(device) | |
| param_info = model.get_param_count() | |
| print_rank0(f"UNet parameters: {param_info['trainable_M']:.2f}M trainable", rank) | |
| # === Load frozen VAE (all ranks need it for perceptual/sync/GAN losses) === | |
| vae = None | |
| try: | |
| from diffusers import AutoencoderKL | |
| vae = AutoencoderKL.from_pretrained( | |
| config.model.vae_model_id, subfolder=config.model.vae_subfolder | |
| ).to(device) | |
| vae.eval() | |
| vae.requires_grad_(False) | |
| print_rank0("VAE loaded for loss computation + visualization", rank) | |
| except Exception as e: | |
| print_rank0(f"VAE not loaded (perceptual/sync/GAN losses disabled): {e}", rank) | |
| # === Loss function === | |
| syncnet = None | |
| syncnet_path = config.model.syncnet.get("checkpoint") if hasattr(config.model, "syncnet") else None | |
| if syncnet_path and Path(syncnet_path).exists(): | |
| from src.models.syncnet import SyncNet | |
| syncnet = SyncNet().to(device) | |
| ckpt = torch.load(syncnet_path, map_location=device, weights_only=True) | |
| state_dict = ckpt["model_state_dict"] | |
| # Handle DDP-saved checkpoints | |
| if any(k.startswith("module.") for k in state_dict.keys()): | |
| state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()} | |
| syncnet.load_state_dict(state_dict) | |
| syncnet.eval() | |
| syncnet.requires_grad_(False) | |
| print_rank0(f"SyncNet loaded from {syncnet_path}", rank) | |
| loss_fn = CompositeLoss( | |
| weight_l1=config.training.loss_weights.l1, | |
| weight_perceptual=config.training.loss_weights.perceptual, | |
| weight_sync=config.training.loss_weights.sync, | |
| weight_gan=config.training.loss_weights.gan, | |
| syncnet=syncnet, | |
| ).to(device) | |
| # === Resume from checkpoint === | |
| start_epoch = 0 | |
| global_step = 0 | |
| best_val_loss = float("inf") | |
| patience_counter = 0 | |
| patience = config.training.get("early_stopping_patience", 0) # 0 = disabled | |
| ckpt = None # Will be set if resuming | |
| if resume_path and os.path.exists(resume_path): | |
| print_rank0(f"Resuming from checkpoint: {resume_path}", rank) | |
| ckpt = torch.load(resume_path, map_location=device, weights_only=False) | |
| # Verify checkpoint integrity | |
| if not verify_checkpoint(ckpt, resume_path, rank): | |
| print_rank0("ERROR: Checkpoint verification failed! Training from scratch.", rank) | |
| ckpt = None | |
| if ckpt is not None: | |
| # For best_model.pt with EMA: use training weights if available (matches optimizer state) | |
| if ckpt.get("has_ema_weights") and "training_model_state_dict" in ckpt: | |
| print_rank0(" Using training weights from best_model.pt (not EMA) for proper resume", rank) | |
| state_dict = ckpt["training_model_state_dict"] | |
| else: | |
| state_dict = ckpt["model_state_dict"] | |
| # Handle both DDP and non-DDP checkpoints | |
| if any(k.startswith("module.") for k in state_dict.keys()): | |
| state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()} | |
| model.load_state_dict(state_dict) | |
| start_epoch = ckpt.get("epoch", 0) + 1 | |
| global_step = ckpt.get("global_step", 0) | |
| best_val_loss = ckpt.get("best_val_loss", float("inf")) | |
| patience_counter = ckpt.get("patience_counter", 0) | |
| # Warn if world_size changed (LR scaling will differ) | |
| saved_world_size = ckpt.get("world_size") | |
| if saved_world_size and saved_world_size != world_size: | |
| print_rank0( | |
| f" WARNING: world_size changed from {saved_world_size} to {world_size}. " | |
| f"Learning rate scaling will differ!", | |
| rank, | |
| ) | |
| print_rank0( | |
| f" Resumed: epoch={start_epoch}, step={global_step}, " | |
| f"best_val_loss={best_val_loss:.4f}, patience={patience_counter}/{patience}", | |
| rank, | |
| ) | |
| elif resume_path: | |
| print_rank0(f"WARNING: Checkpoint not found at {resume_path}, training from scratch", rank) | |
| # === Wrap model with DDP === | |
| # Only the UNet is wrapped since it's the only trainable component. | |
| # find_unused_parameters=False is faster when all params get gradients. | |
| model = DDP(model, device_ids=[local_rank], output_device=local_rank) | |
| # === Optimizer === | |
| # Linear scaling: lr × world_size (gradient_accumulation already accounted for in steps) | |
| scaled_lr = config.training.learning_rate * world_size | |
| print_rank0( | |
| f"Learning rate: {config.training.learning_rate} × {world_size} = {scaled_lr}", rank | |
| ) | |
| optimizer = torch.optim.AdamW( | |
| model.parameters(), | |
| lr=scaled_lr, | |
| weight_decay=config.training.weight_decay, | |
| betas=tuple(config.training.betas), | |
| ) | |
| # Restore optimizer state if resuming | |
| if ckpt is not None and "optimizer_state_dict" in ckpt: | |
| try: | |
| optimizer.load_state_dict(ckpt["optimizer_state_dict"]) | |
| for param_group in optimizer.param_groups: | |
| param_group["lr"] = scaled_lr | |
| print_rank0(" Restored optimizer state", rank) | |
| except Exception as e: | |
| print_rank0(f" Could not restore optimizer: {e}", rank) | |
| # Scheduler | |
| accum_steps = config.training.gradient_accumulation_steps | |
| steps_per_epoch = len(train_loader) // accum_steps | |
| total_steps = steps_per_epoch * config.training.max_epochs | |
| scheduler = create_lr_scheduler(optimizer, config, total_steps, scaled_lr) | |
| # Advance scheduler if resuming | |
| if ckpt is not None and "scheduler_state_dict" in ckpt: | |
| try: | |
| scheduler.load_state_dict(ckpt["scheduler_state_dict"]) | |
| # Fix: LambdaLR.__init__ sets LR to lr_lambda(0) (=0 during warmup). | |
| # load_state_dict restores last_epoch but does NOT update the optimizer LR. | |
| # Manually apply the correct LR now. | |
| for param_group, lr in zip(optimizer.param_groups, scheduler.get_last_lr()): | |
| param_group["lr"] = lr | |
| print_rank0(f" Restored scheduler state (LR={scheduler.get_last_lr()[0]:.2e})", rank) | |
| except Exception: | |
| # Manually advance | |
| steps_done = steps_per_epoch * start_epoch | |
| for _ in range(steps_done): | |
| scheduler.step() | |
| print_rank0(f" Advanced scheduler {steps_done} steps", rank) | |
| # Mixed precision | |
| use_amp = config.training.mixed_precision in ("fp16", "bf16") | |
| scaler = GradScaler(enabled=use_amp and config.training.mixed_precision == "fp16") | |
| amp_dtype = torch.bfloat16 if config.training.mixed_precision == "bf16" else torch.float16 | |
| if ckpt is not None: | |
| if scaler.is_enabled() and ckpt.get("scaler_state_dict"): | |
| scaler.load_state_dict(ckpt["scaler_state_dict"]) | |
| # EMA (operates on unwrapped model) | |
| ema = None | |
| if config.training.ema.get("enabled", False): | |
| # EMA tracks the unwrapped model parameters | |
| ema = EMA( | |
| model.module, | |
| decay=config.training.ema.decay, | |
| update_every=config.training.ema.update_every, | |
| ) | |
| if ckpt is not None and "ema_state_dict" in ckpt: | |
| try: | |
| ema.load_state_dict(ckpt["ema_state_dict"]) | |
| print_rank0(" Restored EMA state", rank) | |
| except Exception as e: | |
| print_rank0(f" Could not restore EMA: {e}", rank) | |
| print_rank0(f"EMA enabled (decay={config.training.ema.decay})", rank) | |
| # Logging (rank 0 only) | |
| writer = None | |
| if rank == 0: | |
| log_dir = Path(config.training.log_dir) | |
| log_dir.mkdir(parents=True, exist_ok=True) | |
| writer = SummaryWriter(log_dir=str(log_dir)) | |
| checkpoint_dir = Path(config.training.checkpoint_dir) | |
| if rank == 0: | |
| checkpoint_dir.mkdir(parents=True, exist_ok=True) | |
| # === Post-resume validation (sanity check) === | |
| if ckpt is not None: | |
| print_rank0("\n--- Post-resume sanity check: running validation ---", rank) | |
| sanity_val_loss = validate_distributed( | |
| model=model, | |
| val_loader=val_loader, | |
| loss_fn=loss_fn, | |
| device=device, | |
| amp_dtype=amp_dtype, | |
| use_amp=use_amp, | |
| writer=writer, | |
| global_step=global_step, | |
| rank=rank, | |
| num_samples=config.training.num_val_samples, | |
| vae=vae, | |
| ) | |
| print_rank0( | |
| f" Sanity check val_loss: {sanity_val_loss:.4f} " | |
| f"(expected ~{best_val_loss:.4f})", | |
| rank, | |
| ) | |
| if sanity_val_loss > best_val_loss * 3: | |
| print_rank0( | |
| f" WARNING: Val loss {sanity_val_loss:.4f} is >3x the best ({best_val_loss:.4f}). " | |
| f"Checkpoint may be corrupt! Consider using an older checkpoint.", | |
| rank, | |
| ) | |
| print_rank0("--- Sanity check complete ---\n", rank) | |
| # Free checkpoint from memory | |
| del ckpt | |
| gc.collect() | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| # === Training Loop === | |
| print_rank0( | |
| f"\nStarting training from epoch {start_epoch + 1} to {config.training.max_epochs}\n" | |
| f"Steps per epoch: {steps_per_epoch} (with {accum_steps}× accumulation)", | |
| rank, | |
| ) | |
| training_start_time = time.time() | |
| for epoch in range(start_epoch, config.training.max_epochs): | |
| model.train() | |
| epoch_loss = 0 | |
| epoch_losses = {} | |
| n_batches = 0 | |
| # CRITICAL: Set epoch for proper shuffling | |
| train_sampler.set_epoch(epoch) | |
| pbar = ( | |
| tqdm(train_loader, desc=f"Epoch {epoch + 1}/{config.training.max_epochs}") | |
| if rank == 0 | |
| else train_loader | |
| ) | |
| epoch_start_time = time.time() | |
| optimizer.zero_grad() | |
| for batch_idx, batch in enumerate(pbar): | |
| target_latent = batch["target_latent"].to(device, non_blocking=True) | |
| ref_latent = batch["reference_latent"].to(device, non_blocking=True) | |
| audio_features = batch["audio_features"].to(device, non_blocking=True) | |
| target_image = batch["target_image"].to(device, non_blocking=True) | |
| mel_segment = batch["mel_segment"].to(device, non_blocking=True) | |
| lip_frames = batch["lip_frames"].to(device, non_blocking=True) | |
| B, C, H, W = target_latent.shape | |
| mask = torch.zeros(B, 1, H, W, device=device) | |
| mask[:, :, H // 2:, :] = 1.0 | |
| masked_latent = target_latent * (1 - mask) | |
| # === Gradient accumulation with DDP sync control === | |
| # Only synchronize gradients on the LAST accumulation step. | |
| # This avoids unnecessary AllReduce during intermediate steps. | |
| is_accumulation_step = ((batch_idx + 1) % accum_steps == 0) or (batch_idx + 1 == len(train_loader)) | |
| # Use no_sync context to skip gradient sync on intermediate steps | |
| context = model.no_sync if not is_accumulation_step else nullcontext | |
| with context(): | |
| with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp): | |
| pred_latent = model( | |
| masked_latent=masked_latent, | |
| reference_latent=ref_latent, | |
| mask=mask, | |
| audio_features=audio_features, | |
| ) | |
| # Decode predicted latents for perceptual/sync/GAN losses | |
| pred_image = None | |
| pred_lip_frames = None | |
| if vae is not None: | |
| with torch.no_grad(): | |
| scale = vae.config.scaling_factor | |
| pred_image = vae.decode(pred_latent / scale).sample | |
| # Resize to match target_image if resolutions differ | |
| target_h = target_image.shape[2] | |
| if pred_image.shape[2] != target_h: | |
| pred_image = F.interpolate( | |
| pred_image, size=(target_h, target_h), | |
| mode="bilinear", align_corners=False, | |
| ) | |
| # Crop lip region from predicted image for SyncNet loss | |
| img_h = pred_image.shape[2] | |
| lip_crop = pred_image[:, :, img_h // 2:, :] | |
| lip_crop = F.interpolate(lip_crop, size=(96, 96), mode="bilinear", align_corners=False) | |
| T = lip_frames.shape[1] | |
| pred_lip_frames = lip_crop.unsqueeze(1).expand(-1, T, -1, -1, -1) | |
| losses = loss_fn( | |
| pred_latent=pred_latent, | |
| target_latent=target_latent, | |
| mask=mask, | |
| pred_image=pred_image, | |
| target_image=target_image, | |
| lip_frames=pred_lip_frames, | |
| mel_segment=mel_segment, | |
| ) | |
| loss = losses["total"] / accum_steps | |
| if scaler.is_enabled(): | |
| scaler.scale(loss).backward() | |
| else: | |
| loss.backward() | |
| # Optimizer step on accumulation boundary | |
| if is_accumulation_step: | |
| if scaler.is_enabled(): | |
| scaler.unscale_(optimizer) | |
| nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| if scaler.is_enabled(): | |
| scaler.step(optimizer) | |
| scaler.update() | |
| else: | |
| optimizer.step() | |
| optimizer.zero_grad() | |
| scheduler.step() | |
| if ema is not None: | |
| ema.update(model.module) | |
| global_step += 1 | |
| # Track losses | |
| epoch_loss += losses["total"].item() | |
| for k, v in losses.items(): | |
| if k != "total": | |
| epoch_losses[k] = epoch_losses.get(k, 0) + v.item() | |
| n_batches += 1 | |
| if rank == 0 and hasattr(pbar, "set_postfix"): | |
| pbar.set_postfix( | |
| loss=f"{losses['total'].item():.4f}", | |
| lr=f"{scheduler.get_last_lr()[0]:.2e}", | |
| ) | |
| # Logging | |
| if rank == 0 and global_step > 0 and global_step % config.training.log_every_n_steps == 0: | |
| writer.add_scalar("train/loss", losses["total"].item(), global_step) | |
| writer.add_scalar("train/lr", scheduler.get_last_lr()[0], global_step) | |
| for k, v in losses.items(): | |
| if k != "total": | |
| writer.add_scalar(f"train/{k}", v.item(), global_step) | |
| # Epoch stats | |
| avg_loss = epoch_loss / max(1, n_batches) | |
| epoch_time = time.time() - epoch_start_time | |
| print_rank0( | |
| f"Epoch {epoch + 1} - Loss: {avg_loss:.4f}, Time: {epoch_time:.1f}s", rank | |
| ) | |
| if rank == 0: | |
| for k, v in epoch_losses.items(): | |
| print(f" {k}: {v / max(1, n_batches):.4f}") | |
| # === Validation === | |
| if (epoch + 1) % config.training.val_every_n_epochs == 0: | |
| if ema is not None: | |
| ema.apply(model.module) | |
| val_loss = validate_distributed( | |
| model=model, | |
| val_loader=val_loader, | |
| loss_fn=loss_fn, | |
| device=device, | |
| amp_dtype=amp_dtype, | |
| use_amp=use_amp, | |
| writer=writer, | |
| global_step=global_step, | |
| rank=rank, | |
| num_samples=config.training.num_val_samples, | |
| vae=vae, | |
| ) | |
| print_rank0(f" Val Loss: {val_loss:.4f}", rank) | |
| if ema is not None: | |
| ema.restore(model.module) | |
| # Save best model (rank 0 only) + early stopping tracking | |
| improved = val_loss < best_val_loss | |
| if rank == 0 and improved: | |
| best_val_loss = val_loss | |
| patience_counter = 0 | |
| # NOTE: EMA is still applied here. Save both EMA and training weights | |
| # so resume can use training weights (matching optimizer state). | |
| state = { | |
| "epoch": epoch, | |
| "global_step": global_step, | |
| "model_state_dict": model.module.state_dict(), # EMA weights (for inference) | |
| "optimizer_state_dict": optimizer.state_dict(), | |
| "scheduler_state_dict": scheduler.state_dict(), | |
| "scaler_state_dict": scaler.state_dict() if scaler.is_enabled() else None, | |
| "best_val_loss": best_val_loss, | |
| "patience_counter": patience_counter, | |
| "world_size": world_size, | |
| "config": OmegaConf.to_container(config), | |
| "has_ema_weights": ema is not None, | |
| } | |
| if ema is not None: | |
| state["ema_state_dict"] = ema.state_dict() | |
| # Also store training weights for proper resume | |
| ema.restore(model.module) | |
| state["training_model_state_dict"] = model.module.state_dict() | |
| ema.apply(model.module) # Re-apply EMA for continued validation | |
| atomic_save(state, checkpoint_dir / "best_model.pt") | |
| print_rank0(f" Saved best model (val_loss={val_loss:.4f})", rank) | |
| elif rank == 0: | |
| patience_counter += 1 | |
| print_rank0( | |
| f" No improvement ({patience_counter}/{patience})" | |
| if patience > 0 else f" No improvement", | |
| rank, | |
| ) | |
| # Early stopping: broadcast decision from rank 0 to all ranks | |
| if patience > 0: | |
| should_stop = torch.tensor( | |
| [1 if (rank == 0 and patience_counter >= patience) else 0], | |
| device=device, | |
| ) | |
| dist.broadcast(should_stop, src=0) | |
| if should_stop.item() == 1: | |
| print_rank0( | |
| f" Early stopping triggered after {patience_counter} validations without improvement.", | |
| rank, | |
| ) | |
| break | |
| # Periodic checkpoint (rank 0) | |
| if rank == 0 and (epoch + 1) % config.training.save_every_n_epochs == 0: | |
| state = { | |
| "epoch": epoch, | |
| "global_step": global_step, | |
| "model_state_dict": model.module.state_dict(), | |
| "optimizer_state_dict": optimizer.state_dict(), | |
| "scheduler_state_dict": scheduler.state_dict(), | |
| "scaler_state_dict": scaler.state_dict() if scaler.is_enabled() else None, | |
| "best_val_loss": best_val_loss, | |
| "patience_counter": patience_counter, | |
| "world_size": world_size, | |
| "config": OmegaConf.to_container(config), | |
| } | |
| if ema is not None: | |
| state["ema_state_dict"] = ema.state_dict() | |
| atomic_save(state, checkpoint_dir / f"checkpoint_epoch_{epoch + 1}.pt") | |
| print_rank0(f" Saved checkpoint: checkpoint_epoch_{epoch + 1}.pt", rank) | |
| # Keep only top-k checkpoints | |
| ckpts = sorted( | |
| checkpoint_dir.glob("checkpoint_epoch_*.pt"), | |
| key=lambda p: p.stat().st_mtime, | |
| reverse=True, | |
| ) | |
| for old_ckpt in ckpts[config.training.save_top_k:]: | |
| old_ckpt.unlink() | |
| # Synchronize | |
| dist.barrier() | |
| # === Save final model (rank 0) === | |
| if rank == 0: | |
| state = { | |
| "epoch": config.training.max_epochs, | |
| "global_step": global_step, | |
| "model_state_dict": model.module.state_dict(), | |
| "optimizer_state_dict": optimizer.state_dict(), | |
| "scheduler_state_dict": scheduler.state_dict(), | |
| "scaler_state_dict": scaler.state_dict() if scaler.is_enabled() else None, | |
| "best_val_loss": best_val_loss, | |
| "patience_counter": patience_counter, | |
| "world_size": world_size, | |
| "config": OmegaConf.to_container(config), | |
| } | |
| if ema is not None: | |
| state["ema_state_dict"] = ema.state_dict() | |
| atomic_save(state, checkpoint_dir / "final_model.pt") | |
| total_time = time.time() - training_start_time | |
| print_rank0( | |
| f"\nTraining complete in {total_time / 3600:.2f} hours. " | |
| f"Best val loss: {best_val_loss:.4f}", | |
| rank, | |
| ) | |
| if rank == 0: | |
| send_training_complete( | |
| model_name="LipSync UNet", | |
| epoch=config.training.max_epochs, | |
| best_val_loss=best_val_loss, | |
| total_hours=total_time / 3600, | |
| ) | |
| if writer: | |
| writer.close() | |
| cleanup_distributed() | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Train LipSync Model (Distributed)") | |
| parser.add_argument("--config", type=str, default="configs/train_distributed.yaml") | |
| parser.add_argument( | |
| "--resume", | |
| type=str, | |
| default=None, | |
| help="Path to checkpoint to resume from (e.g., checkpoints/best_model.pt)", | |
| ) | |
| args = parser.parse_args() | |
| config = OmegaConf.load(args.config) | |
| # Fallback: allow resume path in config for launcher compatibility. | |
| resume_path = args.resume | |
| if not resume_path: | |
| try: | |
| resume_path = config.training.get("resume_from", None) | |
| except Exception: | |
| resume_path = None | |
| train_distributed(config, resume_path=resume_path) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 37.2 kB
- Xet hash:
- c0fac6dbd6139f71a9c04939feb208ac8483c80c9a3ac9a3f9329a14d2223bc8
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.