minhluanquach/unet-training-data / src /training /train_model_distributed.py
minhluanquach's picture
download
raw
37.2 kB
"""
Main LipSync Model Distributed Training Script
===================================================
Multi-GPU distributed training for the main LipSync model using PyTorch DDP.
Features:
- DistributedDataParallel across multiple GPUs
- Gradient accumulation with proper DDP sync control
- EMA (Exponential Moving Average) support
- Resume from single-GPU or multi-GPU checkpoints
- Frozen VAE + Whisper stay on each GPU (small enough to replicate)
- Only UNet gradients are synchronized across GPUs
- Validation with visual samples (rank 0 only)
Usage (single-node, 4 GPUs):
torchrun --nproc_per_node=4 \
-m src.training.train_model_distributed \
--config configs/train_distributed.yaml \
--resume checkpoints/best_model.pt
Usage (multi-node, 2 nodes × 4 GPUs):
# Node 0 (master):
torchrun --nproc_per_node=4 --nnodes=2 --node_rank=0 \
--master_addr=<IP> --master_port=29500 \
-m src.training.train_model_distributed \
--config configs/train_distributed.yaml \
--resume checkpoints/best_model.pt
# Node 1:
torchrun --nproc_per_node=4 --nnodes=2 --node_rank=1 \
--master_addr=<IP> --master_port=29500 \
-m src.training.train_model_distributed \
--config configs/train_distributed.yaml \
--resume checkpoints/best_model.pt
"""
import argparse
import copy
import gc
import os
import tempfile
import time
from contextlib import nullcontext
from pathlib import Path
import torch
import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
from omegaconf import OmegaConf
from torch.amp import autocast, GradScaler
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data import DataLoader, random_split
from torch.utils.data.distributed import DistributedSampler
from torch.utils.tensorboard import SummaryWriter
from torchvision.utils import make_grid
from tqdm import tqdm
from src.data.dataset import LipSyncDataset
from src.models.unet import LipSyncUNet
from src.training.losses import CompositeLoss
from src.training.notify import send_training_complete
# ================================================================
# EMA — Exponential Moving Average
# ================================================================
class EMA:
"""Exponential Moving Average for model parameters.
Maintains a shadow copy of model weights that are updated as:
shadow = decay * shadow + (1 - decay) * current
EMA weights typically produce smoother and better results at inference.
Works with DDP by operating on the unwrapped model.
"""
def __init__(self, model: nn.Module, decay: float = 0.9999, update_every: int = 10):
self.decay = decay
self.update_every = update_every
self.step = 0
self.shadow = {}
self.backup = {}
for name, param in model.named_parameters():
if param.requires_grad:
self.shadow[name] = param.data.clone()
@torch.no_grad()
def update(self, model: nn.Module):
self.step += 1
if self.step % self.update_every != 0:
return
for name, param in model.named_parameters():
if param.requires_grad and name in self.shadow:
self.shadow[name].lerp_(param.data, 1 - self.decay)
def apply(self, model: nn.Module):
"""Apply EMA weights to model (for evaluation)."""
for name, param in model.named_parameters():
if param.requires_grad and name in self.shadow:
self.backup[name] = param.data.clone()
param.data.copy_(self.shadow[name])
def restore(self, model: nn.Module):
"""Restore original weights after evaluation."""
for name, param in model.named_parameters():
if param.requires_grad and name in self.backup:
param.data.copy_(self.backup[name])
self.backup = {}
def state_dict(self):
return {"shadow": self.shadow, "step": self.step, "decay": self.decay}
def load_state_dict(self, state_dict):
self.shadow = state_dict["shadow"]
self.step = state_dict["step"]
self.decay = state_dict["decay"]
# ================================================================
# Utility functions
# ================================================================
def setup_distributed():
"""Initialize distributed process group."""
dist.init_process_group(backend="nccl")
local_rank = int(os.environ["LOCAL_RANK"])
torch.cuda.set_device(local_rank)
return local_rank
def cleanup_distributed():
"""Clean up distributed process group."""
if dist.is_initialized():
dist.destroy_process_group()
def print_rank0(msg, rank=0):
"""Print only on rank 0."""
if rank == 0:
print(msg)
def atomic_save(state, path):
"""Save checkpoint atomically to prevent corruption from pod interrupts.
Writes to a temporary file first, then renames. This ensures the
checkpoint file is either fully written or not present at all.
"""
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
# Write to temp file in same directory (same filesystem for atomic rename)
fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
try:
os.close(fd)
torch.save(state, tmp_path)
# Atomic rename (on POSIX; on Windows this may not be atomic but is still safer)
os.replace(tmp_path, str(path))
except BaseException:
# Clean up temp file on any failure (including KeyboardInterrupt)
if os.path.exists(tmp_path):
os.remove(tmp_path)
raise
def verify_checkpoint(ckpt, resume_path, rank):
"""Verify checkpoint integrity and log diagnostic info."""
required_keys = ["epoch", "global_step", "model_state_dict"]
missing = [k for k in required_keys if k not in ckpt]
if missing:
print_rank0(f"WARNING: Checkpoint missing keys: {missing}", rank)
return False
state_dict = ckpt["model_state_dict"]
num_params = len(state_dict)
total_elements = sum(v.numel() for v in state_dict.values())
# Quick NaN/Inf check on a sample of tensors
has_nan = False
has_inf = False
sample_vals = []
for i, (name, tensor) in enumerate(state_dict.items()):
if torch.isnan(tensor).any():
has_nan = True
print_rank0(f" WARNING: NaN detected in checkpoint param: {name}", rank)
if torch.isinf(tensor).any():
has_inf = True
print_rank0(f" WARNING: Inf detected in checkpoint param: {name}", rank)
if i < 3: # Log first 3 param stats
sample_vals.append(f"{name}: mean={tensor.float().mean():.6f}, std={tensor.float().std():.6f}")
print_rank0(f" Checkpoint: {num_params} params, {total_elements:,} elements", rank)
for sv in sample_vals:
print_rank0(f" {sv}", rank)
print_rank0(
f" Saved at: epoch={ckpt.get('epoch', '?')}, step={ckpt.get('global_step', '?')}, "
f"best_val_loss={ckpt.get('best_val_loss', '?')}",
rank,
)
if ckpt.get("world_size"):
print_rank0(f" Original world_size: {ckpt['world_size']}", rank)
if has_nan or has_inf:
print_rank0(" ERROR: Checkpoint contains NaN/Inf values — likely corrupted!", rank)
return False
return True
def create_lr_scheduler(optimizer, config, total_steps, base_lr):
"""Create learning rate scheduler with warmup."""
warmup_steps = config.training.scheduler.warmup_steps
min_lr = config.training.scheduler.min_lr
def lr_lambda(step):
if step < warmup_steps:
return step / max(1, warmup_steps)
progress = (step - warmup_steps) / max(1, total_steps - warmup_steps)
return max(
min_lr / base_lr,
0.5 * (1 + torch.cos(torch.tensor(progress * 3.14159)).item()),
)
return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)
def validate_distributed(
model: nn.Module,
val_loader: DataLoader,
loss_fn: CompositeLoss,
device: torch.device,
amp_dtype: torch.dtype,
use_amp: bool,
writer,
global_step: int,
rank: int,
num_samples: int = 4,
vae=None,
):
"""Run validation across all GPUs, aggregate results, log on rank 0."""
model.eval()
total_loss = 0
n_batches = 0
sample_preds = []
sample_targets = []
sample_masked = []
with torch.no_grad():
for batch in val_loader:
target_latent = batch["target_latent"].to(device, non_blocking=True)
ref_latent = batch["reference_latent"].to(device, non_blocking=True)
audio_features = batch["audio_features"].to(device, non_blocking=True)
B, C, H, W = target_latent.shape
mask = torch.zeros(B, 1, H, W, device=device)
mask[:, :, H // 2:, :] = 1.0
masked_latent = target_latent * (1 - mask)
with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp):
# Access the unwrapped model for inference
unet = model.module if hasattr(model, "module") else model
pred_latent = unet(
masked_latent=masked_latent,
reference_latent=ref_latent,
mask=mask,
audio_features=audio_features,
)
losses = loss_fn(pred_latent, target_latent, mask)
total_loss += losses["total"].item()
n_batches += 1
# Collect samples for visualization (rank 0 only)
if rank == 0 and len(sample_preds) < num_samples:
sample_preds.append(pred_latent[:1])
sample_targets.append(target_latent[:1])
sample_masked.append(masked_latent[:1])
# Aggregate validation loss across all GPUs
loss_tensor = torch.tensor([total_loss, n_batches], device=device, dtype=torch.float64)
dist.all_reduce(loss_tensor, op=dist.ReduceOp.SUM)
avg_loss = loss_tensor[0].item() / max(1, loss_tensor[1].item())
if rank == 0 and writer is not None:
writer.add_scalar("val/loss", avg_loss, global_step)
# Decode and visualize samples if VAE available
if vae is not None and sample_preds:
try:
pred_latents = torch.cat(sample_preds[:num_samples], dim=0)
target_latents = torch.cat(sample_targets[:num_samples], dim=0)
masked_latents = torch.cat(sample_masked[:num_samples], dim=0)
scale = vae.config.scaling_factor
with torch.no_grad():
pred_imgs = vae.decode(pred_latents / scale).sample
target_imgs = vae.decode(target_latents / scale).sample
masked_imgs = vae.decode(masked_latents / scale).sample
comparison = torch.cat([masked_imgs, pred_imgs, target_imgs], dim=0)
grid = make_grid(
comparison.clamp(-1, 1) * 0.5 + 0.5,
nrow=len(sample_preds),
padding=2,
)
writer.add_image("val/comparison", grid, global_step)
except Exception as e:
print(f"Warning: Could not generate validation images: {e}")
model.train()
return avg_loss
# ================================================================
# Main Training Function
# ================================================================
def train_distributed(config, resume_path=None):
"""Main distributed training loop for the LipSync model."""
# === Distributed Setup ===
local_rank = setup_distributed()
rank = dist.get_rank()
world_size = dist.get_world_size()
device = torch.device(f"cuda:{local_rank}")
print_rank0(f"Distributed training initialized: {world_size} GPUs", rank)
print_rank0(f" Backend: {dist.get_backend()}", rank)
print(f" Rank {rank}/{world_size} → GPU {local_rank} ({torch.cuda.get_device_name(local_rank)})")
# === Dataset ===
dataset = LipSyncDataset(
processed_dir=config.data.processed_dir,
cache_dir=config.data.cache_dir,
image_size=config.data.image_size,
fps=config.data.fps,
mel_step_size=config.data.mel_step_size,
num_audio_frames=config.data.num_audio_frames,
sample_rate=config.data.sample_rate,
)
if len(dataset) == 0:
print_rank0("ERROR: Dataset is empty. Run preprocessing and feature extraction first.", rank)
cleanup_distributed()
return
# Train/val split — fixed seed for all ranks
val_size = max(1, int(len(dataset) * 0.05))
train_size = len(dataset) - val_size
split_generator = torch.Generator().manual_seed(42)
train_dataset, val_dataset = random_split(
dataset, [train_size, val_size], generator=split_generator
)
# === Distributed Samplers ===
train_sampler = DistributedSampler(
train_dataset, num_replicas=world_size, rank=rank, shuffle=True
)
val_sampler = DistributedSampler(
val_dataset, num_replicas=world_size, rank=rank, shuffle=False
)
train_loader = DataLoader(
train_dataset,
batch_size=config.training.batch_size, # per-GPU batch size
shuffle=False,
sampler=train_sampler,
num_workers=config.data.num_workers,
pin_memory=config.data.pin_memory,
drop_last=True,
)
val_loader = DataLoader(
val_dataset,
batch_size=config.training.batch_size,
shuffle=False,
sampler=val_sampler,
num_workers=config.data.num_workers,
pin_memory=config.data.pin_memory,
)
effective_batch_size = config.training.batch_size * world_size * config.training.gradient_accumulation_steps
print_rank0(
f"Train: {len(train_dataset)}, Val: {len(val_dataset)}\n"
f"Per-GPU batch: {config.training.batch_size}, "
f"Accum steps: {config.training.gradient_accumulation_steps}, "
f"Effective batch: {effective_batch_size}",
rank,
)
# === Model (UNet only — trainable) ===
unet_cfg = OmegaConf.to_container(config.model.unet)
model = LipSyncUNet(**unet_cfg).to(device)
param_info = model.get_param_count()
print_rank0(f"UNet parameters: {param_info['trainable_M']:.2f}M trainable", rank)
# === Load frozen VAE (all ranks need it for perceptual/sync/GAN losses) ===
vae = None
try:
from diffusers import AutoencoderKL
vae = AutoencoderKL.from_pretrained(
config.model.vae_model_id, subfolder=config.model.vae_subfolder
).to(device)
vae.eval()
vae.requires_grad_(False)
print_rank0("VAE loaded for loss computation + visualization", rank)
except Exception as e:
print_rank0(f"VAE not loaded (perceptual/sync/GAN losses disabled): {e}", rank)
# === Loss function ===
syncnet = None
syncnet_path = config.model.syncnet.get("checkpoint") if hasattr(config.model, "syncnet") else None
if syncnet_path and Path(syncnet_path).exists():
from src.models.syncnet import SyncNet
syncnet = SyncNet().to(device)
ckpt = torch.load(syncnet_path, map_location=device, weights_only=True)
state_dict = ckpt["model_state_dict"]
# Handle DDP-saved checkpoints
if any(k.startswith("module.") for k in state_dict.keys()):
state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()}
syncnet.load_state_dict(state_dict)
syncnet.eval()
syncnet.requires_grad_(False)
print_rank0(f"SyncNet loaded from {syncnet_path}", rank)
loss_fn = CompositeLoss(
weight_l1=config.training.loss_weights.l1,
weight_perceptual=config.training.loss_weights.perceptual,
weight_sync=config.training.loss_weights.sync,
weight_gan=config.training.loss_weights.gan,
syncnet=syncnet,
).to(device)
# === Resume from checkpoint ===
start_epoch = 0
global_step = 0
best_val_loss = float("inf")
patience_counter = 0
patience = config.training.get("early_stopping_patience", 0) # 0 = disabled
ckpt = None # Will be set if resuming
if resume_path and os.path.exists(resume_path):
print_rank0(f"Resuming from checkpoint: {resume_path}", rank)
ckpt = torch.load(resume_path, map_location=device, weights_only=False)
# Verify checkpoint integrity
if not verify_checkpoint(ckpt, resume_path, rank):
print_rank0("ERROR: Checkpoint verification failed! Training from scratch.", rank)
ckpt = None
if ckpt is not None:
# For best_model.pt with EMA: use training weights if available (matches optimizer state)
if ckpt.get("has_ema_weights") and "training_model_state_dict" in ckpt:
print_rank0(" Using training weights from best_model.pt (not EMA) for proper resume", rank)
state_dict = ckpt["training_model_state_dict"]
else:
state_dict = ckpt["model_state_dict"]
# Handle both DDP and non-DDP checkpoints
if any(k.startswith("module.") for k in state_dict.keys()):
state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()}
model.load_state_dict(state_dict)
start_epoch = ckpt.get("epoch", 0) + 1
global_step = ckpt.get("global_step", 0)
best_val_loss = ckpt.get("best_val_loss", float("inf"))
patience_counter = ckpt.get("patience_counter", 0)
# Warn if world_size changed (LR scaling will differ)
saved_world_size = ckpt.get("world_size")
if saved_world_size and saved_world_size != world_size:
print_rank0(
f" WARNING: world_size changed from {saved_world_size} to {world_size}. "
f"Learning rate scaling will differ!",
rank,
)
print_rank0(
f" Resumed: epoch={start_epoch}, step={global_step}, "
f"best_val_loss={best_val_loss:.4f}, patience={patience_counter}/{patience}",
rank,
)
elif resume_path:
print_rank0(f"WARNING: Checkpoint not found at {resume_path}, training from scratch", rank)
# === Wrap model with DDP ===
# Only the UNet is wrapped since it's the only trainable component.
# find_unused_parameters=False is faster when all params get gradients.
model = DDP(model, device_ids=[local_rank], output_device=local_rank)
# === Optimizer ===
# Linear scaling: lr × world_size (gradient_accumulation already accounted for in steps)
scaled_lr = config.training.learning_rate * world_size
print_rank0(
f"Learning rate: {config.training.learning_rate} × {world_size} = {scaled_lr}", rank
)
optimizer = torch.optim.AdamW(
model.parameters(),
lr=scaled_lr,
weight_decay=config.training.weight_decay,
betas=tuple(config.training.betas),
)
# Restore optimizer state if resuming
if ckpt is not None and "optimizer_state_dict" in ckpt:
try:
optimizer.load_state_dict(ckpt["optimizer_state_dict"])
for param_group in optimizer.param_groups:
param_group["lr"] = scaled_lr
print_rank0(" Restored optimizer state", rank)
except Exception as e:
print_rank0(f" Could not restore optimizer: {e}", rank)
# Scheduler
accum_steps = config.training.gradient_accumulation_steps
steps_per_epoch = len(train_loader) // accum_steps
total_steps = steps_per_epoch * config.training.max_epochs
scheduler = create_lr_scheduler(optimizer, config, total_steps, scaled_lr)
# Advance scheduler if resuming
if ckpt is not None and "scheduler_state_dict" in ckpt:
try:
scheduler.load_state_dict(ckpt["scheduler_state_dict"])
# Fix: LambdaLR.__init__ sets LR to lr_lambda(0) (=0 during warmup).
# load_state_dict restores last_epoch but does NOT update the optimizer LR.
# Manually apply the correct LR now.
for param_group, lr in zip(optimizer.param_groups, scheduler.get_last_lr()):
param_group["lr"] = lr
print_rank0(f" Restored scheduler state (LR={scheduler.get_last_lr()[0]:.2e})", rank)
except Exception:
# Manually advance
steps_done = steps_per_epoch * start_epoch
for _ in range(steps_done):
scheduler.step()
print_rank0(f" Advanced scheduler {steps_done} steps", rank)
# Mixed precision
use_amp = config.training.mixed_precision in ("fp16", "bf16")
scaler = GradScaler(enabled=use_amp and config.training.mixed_precision == "fp16")
amp_dtype = torch.bfloat16 if config.training.mixed_precision == "bf16" else torch.float16
if ckpt is not None:
if scaler.is_enabled() and ckpt.get("scaler_state_dict"):
scaler.load_state_dict(ckpt["scaler_state_dict"])
# EMA (operates on unwrapped model)
ema = None
if config.training.ema.get("enabled", False):
# EMA tracks the unwrapped model parameters
ema = EMA(
model.module,
decay=config.training.ema.decay,
update_every=config.training.ema.update_every,
)
if ckpt is not None and "ema_state_dict" in ckpt:
try:
ema.load_state_dict(ckpt["ema_state_dict"])
print_rank0(" Restored EMA state", rank)
except Exception as e:
print_rank0(f" Could not restore EMA: {e}", rank)
print_rank0(f"EMA enabled (decay={config.training.ema.decay})", rank)
# Logging (rank 0 only)
writer = None
if rank == 0:
log_dir = Path(config.training.log_dir)
log_dir.mkdir(parents=True, exist_ok=True)
writer = SummaryWriter(log_dir=str(log_dir))
checkpoint_dir = Path(config.training.checkpoint_dir)
if rank == 0:
checkpoint_dir.mkdir(parents=True, exist_ok=True)
# === Post-resume validation (sanity check) ===
if ckpt is not None:
print_rank0("\n--- Post-resume sanity check: running validation ---", rank)
sanity_val_loss = validate_distributed(
model=model,
val_loader=val_loader,
loss_fn=loss_fn,
device=device,
amp_dtype=amp_dtype,
use_amp=use_amp,
writer=writer,
global_step=global_step,
rank=rank,
num_samples=config.training.num_val_samples,
vae=vae,
)
print_rank0(
f" Sanity check val_loss: {sanity_val_loss:.4f} "
f"(expected ~{best_val_loss:.4f})",
rank,
)
if sanity_val_loss > best_val_loss * 3:
print_rank0(
f" WARNING: Val loss {sanity_val_loss:.4f} is >3x the best ({best_val_loss:.4f}). "
f"Checkpoint may be corrupt! Consider using an older checkpoint.",
rank,
)
print_rank0("--- Sanity check complete ---\n", rank)
# Free checkpoint from memory
del ckpt
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
# === Training Loop ===
print_rank0(
f"\nStarting training from epoch {start_epoch + 1} to {config.training.max_epochs}\n"
f"Steps per epoch: {steps_per_epoch} (with {accum_steps}× accumulation)",
rank,
)
training_start_time = time.time()
for epoch in range(start_epoch, config.training.max_epochs):
model.train()
epoch_loss = 0
epoch_losses = {}
n_batches = 0
# CRITICAL: Set epoch for proper shuffling
train_sampler.set_epoch(epoch)
pbar = (
tqdm(train_loader, desc=f"Epoch {epoch + 1}/{config.training.max_epochs}")
if rank == 0
else train_loader
)
epoch_start_time = time.time()
optimizer.zero_grad()
for batch_idx, batch in enumerate(pbar):
target_latent = batch["target_latent"].to(device, non_blocking=True)
ref_latent = batch["reference_latent"].to(device, non_blocking=True)
audio_features = batch["audio_features"].to(device, non_blocking=True)
target_image = batch["target_image"].to(device, non_blocking=True)
mel_segment = batch["mel_segment"].to(device, non_blocking=True)
lip_frames = batch["lip_frames"].to(device, non_blocking=True)
B, C, H, W = target_latent.shape
mask = torch.zeros(B, 1, H, W, device=device)
mask[:, :, H // 2:, :] = 1.0
masked_latent = target_latent * (1 - mask)
# === Gradient accumulation with DDP sync control ===
# Only synchronize gradients on the LAST accumulation step.
# This avoids unnecessary AllReduce during intermediate steps.
is_accumulation_step = ((batch_idx + 1) % accum_steps == 0) or (batch_idx + 1 == len(train_loader))
# Use no_sync context to skip gradient sync on intermediate steps
context = model.no_sync if not is_accumulation_step else nullcontext
with context():
with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp):
pred_latent = model(
masked_latent=masked_latent,
reference_latent=ref_latent,
mask=mask,
audio_features=audio_features,
)
# Decode predicted latents for perceptual/sync/GAN losses
pred_image = None
pred_lip_frames = None
if vae is not None:
with torch.no_grad():
scale = vae.config.scaling_factor
pred_image = vae.decode(pred_latent / scale).sample
# Resize to match target_image if resolutions differ
target_h = target_image.shape[2]
if pred_image.shape[2] != target_h:
pred_image = F.interpolate(
pred_image, size=(target_h, target_h),
mode="bilinear", align_corners=False,
)
# Crop lip region from predicted image for SyncNet loss
img_h = pred_image.shape[2]
lip_crop = pred_image[:, :, img_h // 2:, :]
lip_crop = F.interpolate(lip_crop, size=(96, 96), mode="bilinear", align_corners=False)
T = lip_frames.shape[1]
pred_lip_frames = lip_crop.unsqueeze(1).expand(-1, T, -1, -1, -1)
losses = loss_fn(
pred_latent=pred_latent,
target_latent=target_latent,
mask=mask,
pred_image=pred_image,
target_image=target_image,
lip_frames=pred_lip_frames,
mel_segment=mel_segment,
)
loss = losses["total"] / accum_steps
if scaler.is_enabled():
scaler.scale(loss).backward()
else:
loss.backward()
# Optimizer step on accumulation boundary
if is_accumulation_step:
if scaler.is_enabled():
scaler.unscale_(optimizer)
nn.utils.clip_grad_norm_(model.parameters(), 1.0)
if scaler.is_enabled():
scaler.step(optimizer)
scaler.update()
else:
optimizer.step()
optimizer.zero_grad()
scheduler.step()
if ema is not None:
ema.update(model.module)
global_step += 1
# Track losses
epoch_loss += losses["total"].item()
for k, v in losses.items():
if k != "total":
epoch_losses[k] = epoch_losses.get(k, 0) + v.item()
n_batches += 1
if rank == 0 and hasattr(pbar, "set_postfix"):
pbar.set_postfix(
loss=f"{losses['total'].item():.4f}",
lr=f"{scheduler.get_last_lr()[0]:.2e}",
)
# Logging
if rank == 0 and global_step > 0 and global_step % config.training.log_every_n_steps == 0:
writer.add_scalar("train/loss", losses["total"].item(), global_step)
writer.add_scalar("train/lr", scheduler.get_last_lr()[0], global_step)
for k, v in losses.items():
if k != "total":
writer.add_scalar(f"train/{k}", v.item(), global_step)
# Epoch stats
avg_loss = epoch_loss / max(1, n_batches)
epoch_time = time.time() - epoch_start_time
print_rank0(
f"Epoch {epoch + 1} - Loss: {avg_loss:.4f}, Time: {epoch_time:.1f}s", rank
)
if rank == 0:
for k, v in epoch_losses.items():
print(f" {k}: {v / max(1, n_batches):.4f}")
# === Validation ===
if (epoch + 1) % config.training.val_every_n_epochs == 0:
if ema is not None:
ema.apply(model.module)
val_loss = validate_distributed(
model=model,
val_loader=val_loader,
loss_fn=loss_fn,
device=device,
amp_dtype=amp_dtype,
use_amp=use_amp,
writer=writer,
global_step=global_step,
rank=rank,
num_samples=config.training.num_val_samples,
vae=vae,
)
print_rank0(f" Val Loss: {val_loss:.4f}", rank)
if ema is not None:
ema.restore(model.module)
# Save best model (rank 0 only) + early stopping tracking
improved = val_loss < best_val_loss
if rank == 0 and improved:
best_val_loss = val_loss
patience_counter = 0
# NOTE: EMA is still applied here. Save both EMA and training weights
# so resume can use training weights (matching optimizer state).
state = {
"epoch": epoch,
"global_step": global_step,
"model_state_dict": model.module.state_dict(), # EMA weights (for inference)
"optimizer_state_dict": optimizer.state_dict(),
"scheduler_state_dict": scheduler.state_dict(),
"scaler_state_dict": scaler.state_dict() if scaler.is_enabled() else None,
"best_val_loss": best_val_loss,
"patience_counter": patience_counter,
"world_size": world_size,
"config": OmegaConf.to_container(config),
"has_ema_weights": ema is not None,
}
if ema is not None:
state["ema_state_dict"] = ema.state_dict()
# Also store training weights for proper resume
ema.restore(model.module)
state["training_model_state_dict"] = model.module.state_dict()
ema.apply(model.module) # Re-apply EMA for continued validation
atomic_save(state, checkpoint_dir / "best_model.pt")
print_rank0(f" Saved best model (val_loss={val_loss:.4f})", rank)
elif rank == 0:
patience_counter += 1
print_rank0(
f" No improvement ({patience_counter}/{patience})"
if patience > 0 else f" No improvement",
rank,
)
# Early stopping: broadcast decision from rank 0 to all ranks
if patience > 0:
should_stop = torch.tensor(
[1 if (rank == 0 and patience_counter >= patience) else 0],
device=device,
)
dist.broadcast(should_stop, src=0)
if should_stop.item() == 1:
print_rank0(
f" Early stopping triggered after {patience_counter} validations without improvement.",
rank,
)
break
# Periodic checkpoint (rank 0)
if rank == 0 and (epoch + 1) % config.training.save_every_n_epochs == 0:
state = {
"epoch": epoch,
"global_step": global_step,
"model_state_dict": model.module.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"scheduler_state_dict": scheduler.state_dict(),
"scaler_state_dict": scaler.state_dict() if scaler.is_enabled() else None,
"best_val_loss": best_val_loss,
"patience_counter": patience_counter,
"world_size": world_size,
"config": OmegaConf.to_container(config),
}
if ema is not None:
state["ema_state_dict"] = ema.state_dict()
atomic_save(state, checkpoint_dir / f"checkpoint_epoch_{epoch + 1}.pt")
print_rank0(f" Saved checkpoint: checkpoint_epoch_{epoch + 1}.pt", rank)
# Keep only top-k checkpoints
ckpts = sorted(
checkpoint_dir.glob("checkpoint_epoch_*.pt"),
key=lambda p: p.stat().st_mtime,
reverse=True,
)
for old_ckpt in ckpts[config.training.save_top_k:]:
old_ckpt.unlink()
# Synchronize
dist.barrier()
# === Save final model (rank 0) ===
if rank == 0:
state = {
"epoch": config.training.max_epochs,
"global_step": global_step,
"model_state_dict": model.module.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"scheduler_state_dict": scheduler.state_dict(),
"scaler_state_dict": scaler.state_dict() if scaler.is_enabled() else None,
"best_val_loss": best_val_loss,
"patience_counter": patience_counter,
"world_size": world_size,
"config": OmegaConf.to_container(config),
}
if ema is not None:
state["ema_state_dict"] = ema.state_dict()
atomic_save(state, checkpoint_dir / "final_model.pt")
total_time = time.time() - training_start_time
print_rank0(
f"\nTraining complete in {total_time / 3600:.2f} hours. "
f"Best val loss: {best_val_loss:.4f}",
rank,
)
if rank == 0:
send_training_complete(
model_name="LipSync UNet",
epoch=config.training.max_epochs,
best_val_loss=best_val_loss,
total_hours=total_time / 3600,
)
if writer:
writer.close()
cleanup_distributed()
def main():
parser = argparse.ArgumentParser(description="Train LipSync Model (Distributed)")
parser.add_argument("--config", type=str, default="configs/train_distributed.yaml")
parser.add_argument(
"--resume",
type=str,
default=None,
help="Path to checkpoint to resume from (e.g., checkpoints/best_model.pt)",
)
args = parser.parse_args()
config = OmegaConf.load(args.config)
# Fallback: allow resume path in config for launcher compatibility.
resume_path = args.resume
if not resume_path:
try:
resume_path = config.training.get("resume_from", None)
except Exception:
resume_path = None
train_distributed(config, resume_path=resume_path)
if __name__ == "__main__":
main()

Xet Storage Details

Size:
37.2 kB
·
Xet hash:
c0fac6dbd6139f71a9c04939feb208ac8483c80c9a3ac9a3f9329a14d2223bc8

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.