minhluanquach's picture
download
raw
11.1 kB
"""
SyncNet Pre-Training Script
==============================
Pre-trains the SyncNet discriminator on the dataset.
This must be done before main model training.
The SyncNet learns to judge whether audio and lip movements are synchronized.
It's later used as a frozen loss function during main training.
Usage:
python -m src.training.train_syncnet --config configs/syncnet.yaml
"""
import argparse
import os
import time
from pathlib import Path
import torch
import torch.nn as nn
import torch.nn.functional as F
from omegaconf import OmegaConf
from torch.amp import autocast, GradScaler
from torch.utils.data import DataLoader, random_split
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm
from src.data.dataset import SyncNetDataset
from src.models.syncnet import SyncNet
ROOT_DIR = Path(__file__).parent.parent.parent
print(os.listdir(ROOT_DIR))
def train_syncnet(config, resume_checkpoint=None):
"""Main SyncNet training loop."""
device = torch.device(config.hardware.device if torch.cuda.is_available() else "cpu")
print(f"Device: {device}")
joined_path = Path.joinpath(ROOT_DIR, config.data.processed_dir)
print(f"Processed data path: {joined_path}")
# === Dataset ===
dataset = SyncNetDataset(
processed_dir=Path.joinpath(ROOT_DIR, config.data.processed_dir),
image_size=config.model.image_size,
fps=config.data.fps,
num_frames=config.model.num_audio_frames,
mel_step_size=config.model.mel_steps,
sample_rate=config.data.sample_rate,
)
if len(dataset) == 0:
print("ERROR: Dataset is empty. Run preprocessing first.")
return
# Split train/val
val_size = max(1, int(len(dataset) * 0.1))
train_size = len(dataset) - val_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
train_loader = DataLoader(
train_dataset,
batch_size=config.training.batch_size,
shuffle=True,
num_workers=config.data.num_workers,
pin_memory=config.data.pin_memory,
drop_last=True,
)
val_loader = DataLoader(
val_dataset,
batch_size=config.training.batch_size,
shuffle=False,
num_workers=config.data.num_workers,
pin_memory=config.data.pin_memory,
)
print(f"Train: {len(train_dataset)} samples, Val: {len(val_dataset)} samples")
# === Model ===
model = SyncNet(
image_channels=config.model.image_channels,
image_size=config.model.image_size,
embedding_dim=config.model.embedding_dim,
num_frames=config.model.num_audio_frames,
mel_bins=80,
mel_steps=config.model.num_audio_frames * config.model.mel_steps,
).to(device)
total_params = sum(p.numel() for p in model.parameters()) / 1e6
print(f"SyncNet parameters: {total_params:.2f}M")
# === Optimizer ===
optimizer = torch.optim.AdamW(
model.parameters(),
lr=config.training.learning_rate,
weight_decay=config.training.weight_decay,
betas=tuple(config.training.betas),
)
# Learning rate scheduler
total_steps = len(train_loader) * config.training.max_epochs
warmup_steps = config.training.scheduler.warmup_steps
def lr_lambda(step):
if step < warmup_steps:
return step / max(1, warmup_steps)
progress = (step - warmup_steps) / max(1, total_steps - warmup_steps)
return max(config.training.scheduler.min_lr / config.training.learning_rate,
0.5 * (1 + torch.cos(torch.tensor(progress * 3.14159)).item()))
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)
# Mixed precision
use_amp = config.training.mixed_precision in ("fp16", "bf16")
scaler = GradScaler(enabled=use_amp and config.training.mixed_precision == "fp16")
amp_dtype = torch.bfloat16 if config.training.mixed_precision == "bf16" else torch.float16
# Logging
log_dir = Path(config.training.log_dir)
log_dir.mkdir(parents=True, exist_ok=True)
writer = SummaryWriter(log_dir=str(log_dir))
checkpoint_dir = Path(config.training.checkpoint_dir)
checkpoint_dir.mkdir(parents=True, exist_ok=True)
# === Resume from checkpoint ===
start_epoch = 0
global_step = 0
best_val_loss = float("inf")
if resume_checkpoint is not None:
ckpt_path = Path(resume_checkpoint)
if ckpt_path.exists():
print(f"Resuming from checkpoint: {ckpt_path}")
checkpoint = torch.load(ckpt_path, map_location=device, weights_only=False)
model.load_state_dict(checkpoint["model_state_dict"])
optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
start_epoch = checkpoint["epoch"] + 1
if "val_loss" in checkpoint:
best_val_loss = checkpoint["val_loss"]
# Advance scheduler to the correct step
global_step = start_epoch * len(train_loader)
for _ in range(global_step):
scheduler.step()
print(f"Resumed at epoch {start_epoch}, global_step {global_step}, best_val_loss {best_val_loss:.4f}")
else:
print(f"WARNING: Checkpoint not found at {ckpt_path}, training from scratch.")
# === Training Loop ===
for epoch in range(start_epoch, config.training.max_epochs):
model.train()
epoch_loss = 0
epoch_acc = 0
n_batches = 0
pbar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{config.training.max_epochs}")
for batch in pbar:
lip_frames = batch["lip_frames"].to(device) # (B, T, C, H, W)
mel = batch["mel"].to(device) # (B, 1, mel_bins, mel_steps)
labels = batch["label"].to(device) # (B,)
with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp):
output = model(lip_frames, mel)
video_emb = output["video_embedding"]
audio_emb = output["audio_embedding"]
# Contrastive loss
loss = SyncNet.contrastive_loss(
video_emb, audio_emb, model.logit_scale
)
# Backward
optimizer.zero_grad()
if scaler.is_enabled():
scaler.scale(loss).backward()
scaler.unscale_(optimizer)
nn.utils.clip_grad_norm_(model.parameters(), 1.0)
scaler.step(optimizer)
scaler.update()
else:
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
scheduler.step()
# Accuracy (how often top-1 match is correct)
with torch.no_grad():
sim = video_emb @ audio_emb.t()
preds = sim.argmax(dim=1)
targets = torch.arange(len(preds), device=device)
acc = (preds == targets).float().mean().item()
epoch_loss += loss.item()
epoch_acc += acc
n_batches += 1
global_step += 1
pbar.set_postfix(loss=f"{loss.item():.4f}", acc=f"{acc:.3f}")
if global_step % config.training.log_every_n_steps == 0:
writer.add_scalar("train/loss", loss.item(), global_step)
writer.add_scalar("train/accuracy", acc, global_step)
writer.add_scalar("train/lr", scheduler.get_last_lr()[0], global_step)
writer.add_scalar("train/logit_scale", model.logit_scale.exp().item(), global_step)
avg_train_loss = epoch_loss / max(1, n_batches)
avg_train_acc = epoch_acc / max(1, n_batches)
print(f"Epoch {epoch+1} - Train Loss: {avg_train_loss:.4f}, Acc: {avg_train_acc:.3f}")
# === Validation ===
if (epoch + 1) % config.training.val_every_n_epochs == 0:
model.eval()
val_loss = 0
val_acc = 0
n_val = 0
with torch.no_grad():
for batch in val_loader:
lip_frames = batch["lip_frames"].to(device)
mel = batch["mel"].to(device)
with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp):
output = model(lip_frames, mel)
loss = SyncNet.contrastive_loss(
output["video_embedding"],
output["audio_embedding"],
model.logit_scale,
)
sim = output["video_embedding"] @ output["audio_embedding"].t()
preds = sim.argmax(dim=1)
targets = torch.arange(len(preds), device=device)
acc = (preds == targets).float().mean().item()
val_loss += loss.item()
val_acc += acc
n_val += 1
avg_val_loss = val_loss / max(1, n_val)
avg_val_acc = val_acc / max(1, n_val)
print(f" Val Loss: {avg_val_loss:.4f}, Val Acc: {avg_val_acc:.3f}")
writer.add_scalar("val/loss", avg_val_loss, global_step)
writer.add_scalar("val/accuracy", avg_val_acc, global_step)
# Save best
if avg_val_loss < best_val_loss:
best_val_loss = avg_val_loss
torch.save(
{
"epoch": epoch,
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"val_loss": avg_val_loss,
},
checkpoint_dir / "syncnet_best.pt",
)
print(f" Saved best model (val_loss={avg_val_loss:.4f})")
# Save periodic checkpoint
if (epoch + 1) % config.training.save_every_n_epochs == 0:
torch.save(
{
"epoch": epoch,
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
},
checkpoint_dir / f"syncnet_epoch_{epoch+1}.pt",
)
writer.close()
print(f"\nSyncNet training complete. Best val loss: {best_val_loss:.4f}")
def main():
parser = argparse.ArgumentParser(description="Train SyncNet")
parser.add_argument("--config", type=str, default="configs/syncnet.yaml")
parser.add_argument("--resume", type=str, default=None,
help="Path to checkpoint to resume from (e.g. checkpoints/syncnet_best.pt)")
args = parser.parse_args()
config = OmegaConf.load(args.config)
train_syncnet(config, resume_checkpoint=args.resume)
if __name__ == "__main__":
main()

Xet Storage Details

Size:
11.1 kB
·
Xet hash:
7d81c565ae3d456e46697146acde1c8eac5bd9496749a7410c2927f69ee1cacf

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.