Buckets:
| """ | |
| SyncNet Distributed Training Script | |
| ======================================= | |
| Multi-GPU distributed training for SyncNet using PyTorch DDP. | |
| Supports resuming from single-GPU checkpoints and multi-node training. | |
| Usage (single-node, 4 GPUs): | |
| torchrun --nproc_per_node=4 \ | |
| -m src.training.train_syncnet_distributed \ | |
| --config configs/syncnet_distributed.yaml \ | |
| --resume checkpoints/syncnet_best.pt | |
| Usage (multi-node, 2 nodes × 4 GPUs): | |
| # Node 0 (master): | |
| torchrun --nproc_per_node=4 --nnodes=2 --node_rank=0 \ | |
| --master_addr=<IP> --master_port=29500 \ | |
| -m src.training.train_syncnet_distributed \ | |
| --config configs/syncnet_distributed.yaml \ | |
| --resume checkpoints/syncnet_best.pt | |
| # Node 1: | |
| torchrun --nproc_per_node=4 --nnodes=2 --node_rank=1 \ | |
| --master_addr=<IP> --master_port=29500 \ | |
| -m src.training.train_syncnet_distributed \ | |
| --config configs/syncnet_distributed.yaml \ | |
| --resume checkpoints/syncnet_best.pt | |
| """ | |
| import argparse | |
| import os | |
| import time | |
| from pathlib import Path | |
| import torch | |
| import torch.distributed as dist | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from omegaconf import OmegaConf | |
| from torch.amp import autocast, GradScaler | |
| from torch.nn.parallel import DistributedDataParallel as DDP | |
| from torch.utils.data import DataLoader, random_split | |
| from torch.utils.data.distributed import DistributedSampler | |
| from torch.utils.tensorboard import SummaryWriter | |
| from tqdm import tqdm | |
| from src.data.dataset import SyncNetDataset | |
| from src.models.syncnet import SyncNet | |
| from src.training.notify import send_training_complete | |
| ROOT_DIR = Path(__file__).parent.parent.parent | |
| def setup_distributed(): | |
| """Initialize distributed process group. | |
| torchrun sets these environment variables automatically: | |
| - RANK: global rank of this process | |
| - LOCAL_RANK: GPU index on this node | |
| - WORLD_SIZE: total number of processes | |
| - MASTER_ADDR: IP of rank-0 node | |
| - MASTER_PORT: port for rendezvous | |
| """ | |
| dist.init_process_group(backend="nccl") | |
| local_rank = int(os.environ["LOCAL_RANK"]) | |
| torch.cuda.set_device(local_rank) | |
| return local_rank | |
| def cleanup_distributed(): | |
| """Clean up distributed process group.""" | |
| if dist.is_initialized(): | |
| dist.destroy_process_group() | |
| def print_rank0(msg, rank=0): | |
| """Print only on rank 0 to avoid duplicate output.""" | |
| if rank == 0: | |
| print(msg) | |
| def train_syncnet_distributed(config, resume_path=None): | |
| """Main distributed SyncNet training loop.""" | |
| # === Distributed Setup === | |
| local_rank = setup_distributed() | |
| rank = dist.get_rank() | |
| world_size = dist.get_world_size() | |
| device = torch.device(f"cuda:{local_rank}") | |
| print_rank0(f"Distributed training initialized: {world_size} GPUs", rank) | |
| print_rank0(f" Backend: {dist.get_backend()}", rank) | |
| print(f" Rank {rank}/{world_size} → GPU {local_rank} ({torch.cuda.get_device_name(local_rank)})") | |
| # === Dataset === | |
| processed_dir = Path.joinpath(ROOT_DIR, config.data.processed_dir) | |
| print_rank0(f"Processed data path: {processed_dir}", rank) | |
| dataset = SyncNetDataset( | |
| processed_dir=processed_dir, | |
| image_size=config.model.image_size, | |
| fps=config.data.fps, | |
| num_frames=config.model.num_audio_frames, | |
| mel_step_size=config.model.mel_steps, | |
| sample_rate=config.data.sample_rate, | |
| ) | |
| if len(dataset) == 0: | |
| print_rank0("ERROR: Dataset is empty. Run preprocessing first.", rank) | |
| cleanup_distributed() | |
| return | |
| # Split train/val (same split on all ranks due to same seed) | |
| val_size = max(1, int(len(dataset) * 0.1)) | |
| train_size = len(dataset) - val_size | |
| # Use a fixed generator for reproducible splits across all ranks | |
| split_generator = torch.Generator().manual_seed(42) | |
| train_dataset, val_dataset = random_split( | |
| dataset, [train_size, val_size], generator=split_generator | |
| ) | |
| # === Distributed Samplers === | |
| # DistributedSampler shards data across GPUs — each GPU gets a unique subset | |
| train_sampler = DistributedSampler( | |
| train_dataset, num_replicas=world_size, rank=rank, shuffle=True | |
| ) | |
| val_sampler = DistributedSampler( | |
| val_dataset, num_replicas=world_size, rank=rank, shuffle=False | |
| ) | |
| # Note: shuffle=False here because DistributedSampler handles shuffling | |
| train_loader = DataLoader( | |
| train_dataset, | |
| batch_size=config.training.batch_size, # per-GPU batch size | |
| shuffle=False, | |
| sampler=train_sampler, | |
| num_workers=config.data.num_workers, | |
| pin_memory=config.data.pin_memory, | |
| drop_last=True, | |
| ) | |
| val_loader = DataLoader( | |
| val_dataset, | |
| batch_size=config.training.batch_size, | |
| shuffle=False, | |
| sampler=val_sampler, | |
| num_workers=config.data.num_workers, | |
| pin_memory=config.data.pin_memory, | |
| ) | |
| effective_batch_size = config.training.batch_size * world_size | |
| print_rank0( | |
| f"Train: {len(train_dataset)} samples, Val: {len(val_dataset)} samples", rank | |
| ) | |
| print_rank0( | |
| f"Per-GPU batch size: {config.training.batch_size}, " | |
| f"Effective batch size: {effective_batch_size}", | |
| rank, | |
| ) | |
| # === Model === | |
| model = SyncNet( | |
| image_channels=config.model.image_channels, | |
| image_size=config.model.image_size, | |
| embedding_dim=config.model.embedding_dim, | |
| num_frames=config.model.num_audio_frames, | |
| mel_bins=80, | |
| mel_steps=config.model.num_audio_frames * config.model.mel_steps, | |
| ).to(device) | |
| total_params = sum(p.numel() for p in model.parameters()) / 1e6 | |
| print_rank0(f"SyncNet parameters: {total_params:.2f}M", rank) | |
| # === Resume from checkpoint === | |
| start_epoch = 0 | |
| best_val_loss = float("inf") | |
| if resume_path and os.path.exists(resume_path): | |
| print_rank0(f"Resuming from checkpoint: {resume_path}", rank) | |
| # Load checkpoint on rank 0, then broadcast via DDP | |
| checkpoint = torch.load(resume_path, map_location=device, weights_only=False) | |
| # Handle both DDP and non-DDP checkpoints | |
| state_dict = checkpoint["model_state_dict"] | |
| # Remove 'module.' prefix if present (from a DDP checkpoint) | |
| if any(k.startswith("module.") for k in state_dict.keys()): | |
| state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()} | |
| model.load_state_dict(state_dict) | |
| start_epoch = checkpoint.get("epoch", 0) + 1 # Resume from next epoch | |
| best_val_loss = checkpoint.get("val_loss", float("inf")) | |
| print_rank0( | |
| f" Loaded epoch {checkpoint.get('epoch', '?')}, " | |
| f"val_loss={best_val_loss:.4f}", | |
| rank, | |
| ) | |
| elif resume_path: | |
| print_rank0(f"WARNING: Checkpoint not found at {resume_path}, training from scratch", rank) | |
| # === Wrap model with DDP === | |
| # DDP broadcasts model weights from rank 0 to all other ranks | |
| model = DDP(model, device_ids=[local_rank], output_device=local_rank) | |
| # === Optimizer === | |
| # Linear scaling rule: scale LR by world_size to compensate for larger effective batch | |
| scaled_lr = config.training.learning_rate * world_size | |
| print_rank0( | |
| f"Learning rate: {config.training.learning_rate} × {world_size} = {scaled_lr}", | |
| rank, | |
| ) | |
| optimizer = torch.optim.AdamW( | |
| model.parameters(), | |
| lr=scaled_lr, | |
| weight_decay=config.training.weight_decay, | |
| betas=tuple(config.training.betas), | |
| ) | |
| # Restore optimizer state if resuming | |
| if resume_path and os.path.exists(resume_path) and "optimizer_state_dict" in checkpoint: | |
| try: | |
| optimizer.load_state_dict(checkpoint["optimizer_state_dict"]) | |
| # Update LR in restored optimizer to the scaled value | |
| for param_group in optimizer.param_groups: | |
| param_group["lr"] = scaled_lr | |
| print_rank0(" Restored optimizer state", rank) | |
| except Exception as e: | |
| print_rank0(f" Could not restore optimizer state: {e}", rank) | |
| print_rank0(" Starting optimizer from scratch (model weights are still loaded)", rank) | |
| # Learning rate scheduler | |
| total_steps = len(train_loader) * config.training.max_epochs | |
| warmup_steps = config.training.scheduler.warmup_steps | |
| def lr_lambda(step): | |
| if step < warmup_steps: | |
| return step / max(1, warmup_steps) | |
| progress = (step - warmup_steps) / max(1, total_steps - warmup_steps) | |
| return max( | |
| config.training.scheduler.min_lr / scaled_lr, | |
| 0.5 * (1 + torch.cos(torch.tensor(progress * 3.14159)).item()), | |
| ) | |
| scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda) | |
| # Advance scheduler to the correct step if resuming | |
| if start_epoch > 0: | |
| steps_to_skip = len(train_loader) * start_epoch | |
| for _ in range(steps_to_skip): | |
| scheduler.step() | |
| print_rank0(f" Advanced scheduler {steps_to_skip} steps", rank) | |
| # Mixed precision | |
| use_amp = config.training.mixed_precision in ("fp16", "bf16") | |
| scaler = GradScaler(enabled=use_amp and config.training.mixed_precision == "fp16") | |
| amp_dtype = ( | |
| torch.bfloat16 if config.training.mixed_precision == "bf16" else torch.float16 | |
| ) | |
| # Logging (only rank 0) | |
| writer = None | |
| if rank == 0: | |
| log_dir = Path(config.training.log_dir) | |
| log_dir.mkdir(parents=True, exist_ok=True) | |
| writer = SummaryWriter(log_dir=str(log_dir)) | |
| checkpoint_dir = Path(config.training.checkpoint_dir) | |
| if rank == 0: | |
| checkpoint_dir.mkdir(parents=True, exist_ok=True) | |
| # === Training Loop === | |
| global_step = len(train_loader) * start_epoch # Resume global step count | |
| print_rank0( | |
| f"\nStarting training from epoch {start_epoch + 1} to {config.training.max_epochs}", | |
| rank, | |
| ) | |
| print_rank0(f"Total steps per epoch: {len(train_loader)}", rank) | |
| training_start_time = time.time() | |
| for epoch in range(start_epoch, config.training.max_epochs): | |
| model.train() | |
| epoch_loss = 0 | |
| epoch_acc = 0 | |
| n_batches = 0 | |
| # CRITICAL: Set epoch on sampler so data is shuffled differently each epoch | |
| train_sampler.set_epoch(epoch) | |
| pbar = ( | |
| tqdm(train_loader, desc=f"Epoch {epoch + 1}/{config.training.max_epochs}") | |
| if rank == 0 | |
| else train_loader | |
| ) | |
| epoch_start_time = time.time() | |
| for batch in pbar: | |
| lip_frames = batch["lip_frames"].to(device, non_blocking=True) | |
| mel = batch["mel"].to(device, non_blocking=True) | |
| labels = batch["label"].to(device, non_blocking=True) | |
| with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp): | |
| output = model(lip_frames, mel) | |
| video_emb = output["video_embedding"] | |
| audio_emb = output["audio_embedding"] | |
| # Contrastive loss | |
| loss = SyncNet.contrastive_loss( | |
| video_emb, audio_emb, model.module.logit_scale | |
| ) | |
| # Backward | |
| optimizer.zero_grad() | |
| if scaler.is_enabled(): | |
| scaler.scale(loss).backward() | |
| scaler.unscale_(optimizer) | |
| nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| scaler.step(optimizer) | |
| scaler.update() | |
| else: | |
| loss.backward() | |
| nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| optimizer.step() | |
| scheduler.step() | |
| # Accuracy | |
| with torch.no_grad(): | |
| sim = video_emb @ audio_emb.t() | |
| preds = sim.argmax(dim=1) | |
| targets = torch.arange(len(preds), device=device) | |
| acc = (preds == targets).float().mean().item() | |
| epoch_loss += loss.item() | |
| epoch_acc += acc | |
| n_batches += 1 | |
| global_step += 1 | |
| if rank == 0 and hasattr(pbar, "set_postfix"): | |
| pbar.set_postfix(loss=f"{loss.item():.4f}", acc=f"{acc:.3f}") | |
| if rank == 0 and global_step % config.training.log_every_n_steps == 0: | |
| writer.add_scalar("train/loss", loss.item(), global_step) | |
| writer.add_scalar("train/accuracy", acc, global_step) | |
| writer.add_scalar( | |
| "train/lr", scheduler.get_last_lr()[0], global_step | |
| ) | |
| writer.add_scalar( | |
| "train/logit_scale", | |
| model.module.logit_scale.exp().item(), | |
| global_step, | |
| ) | |
| # Epoch stats | |
| avg_train_loss = epoch_loss / max(1, n_batches) | |
| avg_train_acc = epoch_acc / max(1, n_batches) | |
| epoch_time = time.time() - epoch_start_time | |
| print_rank0( | |
| f"Epoch {epoch + 1} - Loss: {avg_train_loss:.4f}, " | |
| f"Acc: {avg_train_acc:.3f}, Time: {epoch_time:.1f}s", | |
| rank, | |
| ) | |
| # === Validation === | |
| if (epoch + 1) % config.training.val_every_n_epochs == 0: | |
| model.eval() | |
| val_loss = 0 | |
| val_acc = 0 | |
| n_val = 0 | |
| with torch.no_grad(): | |
| for batch in val_loader: | |
| lip_frames = batch["lip_frames"].to(device, non_blocking=True) | |
| mel = batch["mel"].to(device, non_blocking=True) | |
| with autocast( | |
| device_type="cuda", dtype=amp_dtype, enabled=use_amp | |
| ): | |
| output = model(lip_frames, mel) | |
| loss = SyncNet.contrastive_loss( | |
| output["video_embedding"], | |
| output["audio_embedding"], | |
| model.module.logit_scale, | |
| ) | |
| sim = output["video_embedding"] @ output["audio_embedding"].t() | |
| preds = sim.argmax(dim=1) | |
| targets = torch.arange(len(preds), device=device) | |
| acc = (preds == targets).float().mean().item() | |
| val_loss += loss.item() | |
| val_acc += acc | |
| n_val += 1 | |
| # Aggregate validation results across GPUs | |
| val_loss_tensor = torch.tensor([val_loss, n_val], device=device) | |
| val_acc_tensor = torch.tensor([val_acc, n_val], device=device) | |
| dist.all_reduce(val_loss_tensor, op=dist.ReduceOp.SUM) | |
| dist.all_reduce(val_acc_tensor, op=dist.ReduceOp.SUM) | |
| avg_val_loss = val_loss_tensor[0].item() / max(1, val_loss_tensor[1].item()) | |
| avg_val_acc = val_acc_tensor[0].item() / max(1, val_acc_tensor[1].item()) | |
| print_rank0( | |
| f" Val Loss: {avg_val_loss:.4f}, Val Acc: {avg_val_acc:.3f}", rank | |
| ) | |
| if rank == 0: | |
| writer.add_scalar("val/loss", avg_val_loss, global_step) | |
| writer.add_scalar("val/accuracy", avg_val_acc, global_step) | |
| # Save best model | |
| if avg_val_loss < best_val_loss: | |
| best_val_loss = avg_val_loss | |
| torch.save( | |
| { | |
| "epoch": epoch, | |
| "model_state_dict": model.module.state_dict(), | |
| "optimizer_state_dict": optimizer.state_dict(), | |
| "val_loss": avg_val_loss, | |
| "world_size": world_size, | |
| "config": OmegaConf.to_container(config), | |
| }, | |
| checkpoint_dir / "syncnet_best.pt", | |
| ) | |
| print_rank0( | |
| f" Saved best model (val_loss={avg_val_loss:.4f})", rank | |
| ) | |
| # Save periodic checkpoint (only rank 0) | |
| if rank == 0 and (epoch + 1) % config.training.save_every_n_epochs == 0: | |
| torch.save( | |
| { | |
| "epoch": epoch, | |
| "model_state_dict": model.module.state_dict(), | |
| "optimizer_state_dict": optimizer.state_dict(), | |
| "val_loss": best_val_loss, | |
| "world_size": world_size, | |
| "config": OmegaConf.to_container(config), | |
| }, | |
| checkpoint_dir / f"syncnet_epoch_{epoch + 1}.pt", | |
| ) | |
| print_rank0(f" Saved checkpoint: syncnet_epoch_{epoch + 1}.pt", rank) | |
| # Synchronize all processes at end of each epoch | |
| dist.barrier() | |
| # === Done === | |
| total_time = time.time() - training_start_time | |
| print_rank0( | |
| f"\nTraining complete in {total_time / 3600:.2f} hours. " | |
| f"Best val loss: {best_val_loss:.4f}", | |
| rank, | |
| ) | |
| if rank == 0: | |
| send_training_complete( | |
| model_name="SyncNet", | |
| epoch=config.training.max_epochs, | |
| best_val_loss=best_val_loss, | |
| total_hours=total_time / 3600, | |
| ) | |
| if writer: | |
| writer.close() | |
| cleanup_distributed() | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Train SyncNet (Distributed)") | |
| parser.add_argument("--config", type=str, default="configs/syncnet_distributed.yaml") | |
| parser.add_argument( | |
| "--resume", | |
| type=str, | |
| default=None, | |
| help="Path to checkpoint to resume from (e.g., checkpoints/syncnet_best.pt)", | |
| ) | |
| args = parser.parse_args() | |
| config = OmegaConf.load(args.config) | |
| train_syncnet_distributed(config, resume_path=args.resume) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 18.3 kB
- Xet hash:
- e0dae68673103c02e9d09461b85805bddb1e23593f224e46e7225e7ed8383882
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.