minhluanquach/unet-training-data / src /training /train_syncnet_distributed.py
minhluanquach's picture
download
raw
18.3 kB
"""
SyncNet Distributed Training Script
=======================================
Multi-GPU distributed training for SyncNet using PyTorch DDP.
Supports resuming from single-GPU checkpoints and multi-node training.
Usage (single-node, 4 GPUs):
torchrun --nproc_per_node=4 \
-m src.training.train_syncnet_distributed \
--config configs/syncnet_distributed.yaml \
--resume checkpoints/syncnet_best.pt
Usage (multi-node, 2 nodes × 4 GPUs):
# Node 0 (master):
torchrun --nproc_per_node=4 --nnodes=2 --node_rank=0 \
--master_addr=<IP> --master_port=29500 \
-m src.training.train_syncnet_distributed \
--config configs/syncnet_distributed.yaml \
--resume checkpoints/syncnet_best.pt
# Node 1:
torchrun --nproc_per_node=4 --nnodes=2 --node_rank=1 \
--master_addr=<IP> --master_port=29500 \
-m src.training.train_syncnet_distributed \
--config configs/syncnet_distributed.yaml \
--resume checkpoints/syncnet_best.pt
"""
import argparse
import os
import time
from pathlib import Path
import torch
import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
from omegaconf import OmegaConf
from torch.amp import autocast, GradScaler
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data import DataLoader, random_split
from torch.utils.data.distributed import DistributedSampler
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm
from src.data.dataset import SyncNetDataset
from src.models.syncnet import SyncNet
from src.training.notify import send_training_complete
ROOT_DIR = Path(__file__).parent.parent.parent
def setup_distributed():
"""Initialize distributed process group.
torchrun sets these environment variables automatically:
- RANK: global rank of this process
- LOCAL_RANK: GPU index on this node
- WORLD_SIZE: total number of processes
- MASTER_ADDR: IP of rank-0 node
- MASTER_PORT: port for rendezvous
"""
dist.init_process_group(backend="nccl")
local_rank = int(os.environ["LOCAL_RANK"])
torch.cuda.set_device(local_rank)
return local_rank
def cleanup_distributed():
"""Clean up distributed process group."""
if dist.is_initialized():
dist.destroy_process_group()
def print_rank0(msg, rank=0):
"""Print only on rank 0 to avoid duplicate output."""
if rank == 0:
print(msg)
def train_syncnet_distributed(config, resume_path=None):
"""Main distributed SyncNet training loop."""
# === Distributed Setup ===
local_rank = setup_distributed()
rank = dist.get_rank()
world_size = dist.get_world_size()
device = torch.device(f"cuda:{local_rank}")
print_rank0(f"Distributed training initialized: {world_size} GPUs", rank)
print_rank0(f" Backend: {dist.get_backend()}", rank)
print(f" Rank {rank}/{world_size} → GPU {local_rank} ({torch.cuda.get_device_name(local_rank)})")
# === Dataset ===
processed_dir = Path.joinpath(ROOT_DIR, config.data.processed_dir)
print_rank0(f"Processed data path: {processed_dir}", rank)
dataset = SyncNetDataset(
processed_dir=processed_dir,
image_size=config.model.image_size,
fps=config.data.fps,
num_frames=config.model.num_audio_frames,
mel_step_size=config.model.mel_steps,
sample_rate=config.data.sample_rate,
)
if len(dataset) == 0:
print_rank0("ERROR: Dataset is empty. Run preprocessing first.", rank)
cleanup_distributed()
return
# Split train/val (same split on all ranks due to same seed)
val_size = max(1, int(len(dataset) * 0.1))
train_size = len(dataset) - val_size
# Use a fixed generator for reproducible splits across all ranks
split_generator = torch.Generator().manual_seed(42)
train_dataset, val_dataset = random_split(
dataset, [train_size, val_size], generator=split_generator
)
# === Distributed Samplers ===
# DistributedSampler shards data across GPUs — each GPU gets a unique subset
train_sampler = DistributedSampler(
train_dataset, num_replicas=world_size, rank=rank, shuffle=True
)
val_sampler = DistributedSampler(
val_dataset, num_replicas=world_size, rank=rank, shuffle=False
)
# Note: shuffle=False here because DistributedSampler handles shuffling
train_loader = DataLoader(
train_dataset,
batch_size=config.training.batch_size, # per-GPU batch size
shuffle=False,
sampler=train_sampler,
num_workers=config.data.num_workers,
pin_memory=config.data.pin_memory,
drop_last=True,
)
val_loader = DataLoader(
val_dataset,
batch_size=config.training.batch_size,
shuffle=False,
sampler=val_sampler,
num_workers=config.data.num_workers,
pin_memory=config.data.pin_memory,
)
effective_batch_size = config.training.batch_size * world_size
print_rank0(
f"Train: {len(train_dataset)} samples, Val: {len(val_dataset)} samples", rank
)
print_rank0(
f"Per-GPU batch size: {config.training.batch_size}, "
f"Effective batch size: {effective_batch_size}",
rank,
)
# === Model ===
model = SyncNet(
image_channels=config.model.image_channels,
image_size=config.model.image_size,
embedding_dim=config.model.embedding_dim,
num_frames=config.model.num_audio_frames,
mel_bins=80,
mel_steps=config.model.num_audio_frames * config.model.mel_steps,
).to(device)
total_params = sum(p.numel() for p in model.parameters()) / 1e6
print_rank0(f"SyncNet parameters: {total_params:.2f}M", rank)
# === Resume from checkpoint ===
start_epoch = 0
best_val_loss = float("inf")
if resume_path and os.path.exists(resume_path):
print_rank0(f"Resuming from checkpoint: {resume_path}", rank)
# Load checkpoint on rank 0, then broadcast via DDP
checkpoint = torch.load(resume_path, map_location=device, weights_only=False)
# Handle both DDP and non-DDP checkpoints
state_dict = checkpoint["model_state_dict"]
# Remove 'module.' prefix if present (from a DDP checkpoint)
if any(k.startswith("module.") for k in state_dict.keys()):
state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()}
model.load_state_dict(state_dict)
start_epoch = checkpoint.get("epoch", 0) + 1 # Resume from next epoch
best_val_loss = checkpoint.get("val_loss", float("inf"))
print_rank0(
f" Loaded epoch {checkpoint.get('epoch', '?')}, "
f"val_loss={best_val_loss:.4f}",
rank,
)
elif resume_path:
print_rank0(f"WARNING: Checkpoint not found at {resume_path}, training from scratch", rank)
# === Wrap model with DDP ===
# DDP broadcasts model weights from rank 0 to all other ranks
model = DDP(model, device_ids=[local_rank], output_device=local_rank)
# === Optimizer ===
# Linear scaling rule: scale LR by world_size to compensate for larger effective batch
scaled_lr = config.training.learning_rate * world_size
print_rank0(
f"Learning rate: {config.training.learning_rate} × {world_size} = {scaled_lr}",
rank,
)
optimizer = torch.optim.AdamW(
model.parameters(),
lr=scaled_lr,
weight_decay=config.training.weight_decay,
betas=tuple(config.training.betas),
)
# Restore optimizer state if resuming
if resume_path and os.path.exists(resume_path) and "optimizer_state_dict" in checkpoint:
try:
optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
# Update LR in restored optimizer to the scaled value
for param_group in optimizer.param_groups:
param_group["lr"] = scaled_lr
print_rank0(" Restored optimizer state", rank)
except Exception as e:
print_rank0(f" Could not restore optimizer state: {e}", rank)
print_rank0(" Starting optimizer from scratch (model weights are still loaded)", rank)
# Learning rate scheduler
total_steps = len(train_loader) * config.training.max_epochs
warmup_steps = config.training.scheduler.warmup_steps
def lr_lambda(step):
if step < warmup_steps:
return step / max(1, warmup_steps)
progress = (step - warmup_steps) / max(1, total_steps - warmup_steps)
return max(
config.training.scheduler.min_lr / scaled_lr,
0.5 * (1 + torch.cos(torch.tensor(progress * 3.14159)).item()),
)
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)
# Advance scheduler to the correct step if resuming
if start_epoch > 0:
steps_to_skip = len(train_loader) * start_epoch
for _ in range(steps_to_skip):
scheduler.step()
print_rank0(f" Advanced scheduler {steps_to_skip} steps", rank)
# Mixed precision
use_amp = config.training.mixed_precision in ("fp16", "bf16")
scaler = GradScaler(enabled=use_amp and config.training.mixed_precision == "fp16")
amp_dtype = (
torch.bfloat16 if config.training.mixed_precision == "bf16" else torch.float16
)
# Logging (only rank 0)
writer = None
if rank == 0:
log_dir = Path(config.training.log_dir)
log_dir.mkdir(parents=True, exist_ok=True)
writer = SummaryWriter(log_dir=str(log_dir))
checkpoint_dir = Path(config.training.checkpoint_dir)
if rank == 0:
checkpoint_dir.mkdir(parents=True, exist_ok=True)
# === Training Loop ===
global_step = len(train_loader) * start_epoch # Resume global step count
print_rank0(
f"\nStarting training from epoch {start_epoch + 1} to {config.training.max_epochs}",
rank,
)
print_rank0(f"Total steps per epoch: {len(train_loader)}", rank)
training_start_time = time.time()
for epoch in range(start_epoch, config.training.max_epochs):
model.train()
epoch_loss = 0
epoch_acc = 0
n_batches = 0
# CRITICAL: Set epoch on sampler so data is shuffled differently each epoch
train_sampler.set_epoch(epoch)
pbar = (
tqdm(train_loader, desc=f"Epoch {epoch + 1}/{config.training.max_epochs}")
if rank == 0
else train_loader
)
epoch_start_time = time.time()
for batch in pbar:
lip_frames = batch["lip_frames"].to(device, non_blocking=True)
mel = batch["mel"].to(device, non_blocking=True)
labels = batch["label"].to(device, non_blocking=True)
with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp):
output = model(lip_frames, mel)
video_emb = output["video_embedding"]
audio_emb = output["audio_embedding"]
# Contrastive loss
loss = SyncNet.contrastive_loss(
video_emb, audio_emb, model.module.logit_scale
)
# Backward
optimizer.zero_grad()
if scaler.is_enabled():
scaler.scale(loss).backward()
scaler.unscale_(optimizer)
nn.utils.clip_grad_norm_(model.parameters(), 1.0)
scaler.step(optimizer)
scaler.update()
else:
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
scheduler.step()
# Accuracy
with torch.no_grad():
sim = video_emb @ audio_emb.t()
preds = sim.argmax(dim=1)
targets = torch.arange(len(preds), device=device)
acc = (preds == targets).float().mean().item()
epoch_loss += loss.item()
epoch_acc += acc
n_batches += 1
global_step += 1
if rank == 0 and hasattr(pbar, "set_postfix"):
pbar.set_postfix(loss=f"{loss.item():.4f}", acc=f"{acc:.3f}")
if rank == 0 and global_step % config.training.log_every_n_steps == 0:
writer.add_scalar("train/loss", loss.item(), global_step)
writer.add_scalar("train/accuracy", acc, global_step)
writer.add_scalar(
"train/lr", scheduler.get_last_lr()[0], global_step
)
writer.add_scalar(
"train/logit_scale",
model.module.logit_scale.exp().item(),
global_step,
)
# Epoch stats
avg_train_loss = epoch_loss / max(1, n_batches)
avg_train_acc = epoch_acc / max(1, n_batches)
epoch_time = time.time() - epoch_start_time
print_rank0(
f"Epoch {epoch + 1} - Loss: {avg_train_loss:.4f}, "
f"Acc: {avg_train_acc:.3f}, Time: {epoch_time:.1f}s",
rank,
)
# === Validation ===
if (epoch + 1) % config.training.val_every_n_epochs == 0:
model.eval()
val_loss = 0
val_acc = 0
n_val = 0
with torch.no_grad():
for batch in val_loader:
lip_frames = batch["lip_frames"].to(device, non_blocking=True)
mel = batch["mel"].to(device, non_blocking=True)
with autocast(
device_type="cuda", dtype=amp_dtype, enabled=use_amp
):
output = model(lip_frames, mel)
loss = SyncNet.contrastive_loss(
output["video_embedding"],
output["audio_embedding"],
model.module.logit_scale,
)
sim = output["video_embedding"] @ output["audio_embedding"].t()
preds = sim.argmax(dim=1)
targets = torch.arange(len(preds), device=device)
acc = (preds == targets).float().mean().item()
val_loss += loss.item()
val_acc += acc
n_val += 1
# Aggregate validation results across GPUs
val_loss_tensor = torch.tensor([val_loss, n_val], device=device)
val_acc_tensor = torch.tensor([val_acc, n_val], device=device)
dist.all_reduce(val_loss_tensor, op=dist.ReduceOp.SUM)
dist.all_reduce(val_acc_tensor, op=dist.ReduceOp.SUM)
avg_val_loss = val_loss_tensor[0].item() / max(1, val_loss_tensor[1].item())
avg_val_acc = val_acc_tensor[0].item() / max(1, val_acc_tensor[1].item())
print_rank0(
f" Val Loss: {avg_val_loss:.4f}, Val Acc: {avg_val_acc:.3f}", rank
)
if rank == 0:
writer.add_scalar("val/loss", avg_val_loss, global_step)
writer.add_scalar("val/accuracy", avg_val_acc, global_step)
# Save best model
if avg_val_loss < best_val_loss:
best_val_loss = avg_val_loss
torch.save(
{
"epoch": epoch,
"model_state_dict": model.module.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"val_loss": avg_val_loss,
"world_size": world_size,
"config": OmegaConf.to_container(config),
},
checkpoint_dir / "syncnet_best.pt",
)
print_rank0(
f" Saved best model (val_loss={avg_val_loss:.4f})", rank
)
# Save periodic checkpoint (only rank 0)
if rank == 0 and (epoch + 1) % config.training.save_every_n_epochs == 0:
torch.save(
{
"epoch": epoch,
"model_state_dict": model.module.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"val_loss": best_val_loss,
"world_size": world_size,
"config": OmegaConf.to_container(config),
},
checkpoint_dir / f"syncnet_epoch_{epoch + 1}.pt",
)
print_rank0(f" Saved checkpoint: syncnet_epoch_{epoch + 1}.pt", rank)
# Synchronize all processes at end of each epoch
dist.barrier()
# === Done ===
total_time = time.time() - training_start_time
print_rank0(
f"\nTraining complete in {total_time / 3600:.2f} hours. "
f"Best val loss: {best_val_loss:.4f}",
rank,
)
if rank == 0:
send_training_complete(
model_name="SyncNet",
epoch=config.training.max_epochs,
best_val_loss=best_val_loss,
total_hours=total_time / 3600,
)
if writer:
writer.close()
cleanup_distributed()
def main():
parser = argparse.ArgumentParser(description="Train SyncNet (Distributed)")
parser.add_argument("--config", type=str, default="configs/syncnet_distributed.yaml")
parser.add_argument(
"--resume",
type=str,
default=None,
help="Path to checkpoint to resume from (e.g., checkpoints/syncnet_best.pt)",
)
args = parser.parse_args()
config = OmegaConf.load(args.config)
train_syncnet_distributed(config, resume_path=args.resume)
if __name__ == "__main__":
main()

Xet Storage Details

Size:
18.3 kB
·
Xet hash:
e0dae68673103c02e9d09461b85805bddb1e23593f224e46e7225e7ed8383882

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.