Buckets:
| """ | |
| SyncNet Pre-Training Script | |
| ============================== | |
| Pre-trains the SyncNet discriminator on the dataset. | |
| This must be done before main model training. | |
| The SyncNet learns to judge whether audio and lip movements are synchronized. | |
| It's later used as a frozen loss function during main training. | |
| Usage: | |
| python -m src.training.train_syncnet --config configs/syncnet.yaml | |
| """ | |
| import argparse | |
| import os | |
| import time | |
| from pathlib import Path | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from omegaconf import OmegaConf | |
| from torch.amp import autocast, GradScaler | |
| from torch.utils.data import DataLoader, random_split | |
| from torch.utils.tensorboard import SummaryWriter | |
| from tqdm import tqdm | |
| from src.data.dataset import SyncNetDataset | |
| from src.models.syncnet import SyncNet | |
| ROOT_DIR = Path(__file__).parent.parent.parent | |
| print(os.listdir(ROOT_DIR)) | |
| def train_syncnet(config, resume_checkpoint=None): | |
| """Main SyncNet training loop.""" | |
| device = torch.device(config.hardware.device if torch.cuda.is_available() else "cpu") | |
| print(f"Device: {device}") | |
| joined_path = Path.joinpath(ROOT_DIR, config.data.processed_dir) | |
| print(f"Processed data path: {joined_path}") | |
| # === Dataset === | |
| dataset = SyncNetDataset( | |
| processed_dir=Path.joinpath(ROOT_DIR, config.data.processed_dir), | |
| image_size=config.model.image_size, | |
| fps=config.data.fps, | |
| num_frames=config.model.num_audio_frames, | |
| mel_step_size=config.model.mel_steps, | |
| sample_rate=config.data.sample_rate, | |
| ) | |
| if len(dataset) == 0: | |
| print("ERROR: Dataset is empty. Run preprocessing first.") | |
| return | |
| # Split train/val | |
| val_size = max(1, int(len(dataset) * 0.1)) | |
| train_size = len(dataset) - val_size | |
| train_dataset, val_dataset = random_split(dataset, [train_size, val_size]) | |
| train_loader = DataLoader( | |
| train_dataset, | |
| batch_size=config.training.batch_size, | |
| shuffle=True, | |
| num_workers=config.data.num_workers, | |
| pin_memory=config.data.pin_memory, | |
| drop_last=True, | |
| ) | |
| val_loader = DataLoader( | |
| val_dataset, | |
| batch_size=config.training.batch_size, | |
| shuffle=False, | |
| num_workers=config.data.num_workers, | |
| pin_memory=config.data.pin_memory, | |
| ) | |
| print(f"Train: {len(train_dataset)} samples, Val: {len(val_dataset)} samples") | |
| # === Model === | |
| model = SyncNet( | |
| image_channels=config.model.image_channels, | |
| image_size=config.model.image_size, | |
| embedding_dim=config.model.embedding_dim, | |
| num_frames=config.model.num_audio_frames, | |
| mel_bins=80, | |
| mel_steps=config.model.num_audio_frames * config.model.mel_steps, | |
| ).to(device) | |
| total_params = sum(p.numel() for p in model.parameters()) / 1e6 | |
| print(f"SyncNet parameters: {total_params:.2f}M") | |
| # === Optimizer === | |
| optimizer = torch.optim.AdamW( | |
| model.parameters(), | |
| lr=config.training.learning_rate, | |
| weight_decay=config.training.weight_decay, | |
| betas=tuple(config.training.betas), | |
| ) | |
| # Learning rate scheduler | |
| total_steps = len(train_loader) * config.training.max_epochs | |
| warmup_steps = config.training.scheduler.warmup_steps | |
| def lr_lambda(step): | |
| if step < warmup_steps: | |
| return step / max(1, warmup_steps) | |
| progress = (step - warmup_steps) / max(1, total_steps - warmup_steps) | |
| return max(config.training.scheduler.min_lr / config.training.learning_rate, | |
| 0.5 * (1 + torch.cos(torch.tensor(progress * 3.14159)).item())) | |
| scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda) | |
| # Mixed precision | |
| use_amp = config.training.mixed_precision in ("fp16", "bf16") | |
| scaler = GradScaler(enabled=use_amp and config.training.mixed_precision == "fp16") | |
| amp_dtype = torch.bfloat16 if config.training.mixed_precision == "bf16" else torch.float16 | |
| # Logging | |
| log_dir = Path(config.training.log_dir) | |
| log_dir.mkdir(parents=True, exist_ok=True) | |
| writer = SummaryWriter(log_dir=str(log_dir)) | |
| checkpoint_dir = Path(config.training.checkpoint_dir) | |
| checkpoint_dir.mkdir(parents=True, exist_ok=True) | |
| # === Resume from checkpoint === | |
| start_epoch = 0 | |
| global_step = 0 | |
| best_val_loss = float("inf") | |
| if resume_checkpoint is not None: | |
| ckpt_path = Path(resume_checkpoint) | |
| if ckpt_path.exists(): | |
| print(f"Resuming from checkpoint: {ckpt_path}") | |
| checkpoint = torch.load(ckpt_path, map_location=device, weights_only=False) | |
| model.load_state_dict(checkpoint["model_state_dict"]) | |
| optimizer.load_state_dict(checkpoint["optimizer_state_dict"]) | |
| start_epoch = checkpoint["epoch"] + 1 | |
| if "val_loss" in checkpoint: | |
| best_val_loss = checkpoint["val_loss"] | |
| # Advance scheduler to the correct step | |
| global_step = start_epoch * len(train_loader) | |
| for _ in range(global_step): | |
| scheduler.step() | |
| print(f"Resumed at epoch {start_epoch}, global_step {global_step}, best_val_loss {best_val_loss:.4f}") | |
| else: | |
| print(f"WARNING: Checkpoint not found at {ckpt_path}, training from scratch.") | |
| # === Training Loop === | |
| for epoch in range(start_epoch, config.training.max_epochs): | |
| model.train() | |
| epoch_loss = 0 | |
| epoch_acc = 0 | |
| n_batches = 0 | |
| pbar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{config.training.max_epochs}") | |
| for batch in pbar: | |
| lip_frames = batch["lip_frames"].to(device) # (B, T, C, H, W) | |
| mel = batch["mel"].to(device) # (B, 1, mel_bins, mel_steps) | |
| labels = batch["label"].to(device) # (B,) | |
| with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp): | |
| output = model(lip_frames, mel) | |
| video_emb = output["video_embedding"] | |
| audio_emb = output["audio_embedding"] | |
| # Contrastive loss | |
| loss = SyncNet.contrastive_loss( | |
| video_emb, audio_emb, model.logit_scale | |
| ) | |
| # Backward | |
| optimizer.zero_grad() | |
| if scaler.is_enabled(): | |
| scaler.scale(loss).backward() | |
| scaler.unscale_(optimizer) | |
| nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| scaler.step(optimizer) | |
| scaler.update() | |
| else: | |
| loss.backward() | |
| nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| optimizer.step() | |
| scheduler.step() | |
| # Accuracy (how often top-1 match is correct) | |
| with torch.no_grad(): | |
| sim = video_emb @ audio_emb.t() | |
| preds = sim.argmax(dim=1) | |
| targets = torch.arange(len(preds), device=device) | |
| acc = (preds == targets).float().mean().item() | |
| epoch_loss += loss.item() | |
| epoch_acc += acc | |
| n_batches += 1 | |
| global_step += 1 | |
| pbar.set_postfix(loss=f"{loss.item():.4f}", acc=f"{acc:.3f}") | |
| if global_step % config.training.log_every_n_steps == 0: | |
| writer.add_scalar("train/loss", loss.item(), global_step) | |
| writer.add_scalar("train/accuracy", acc, global_step) | |
| writer.add_scalar("train/lr", scheduler.get_last_lr()[0], global_step) | |
| writer.add_scalar("train/logit_scale", model.logit_scale.exp().item(), global_step) | |
| avg_train_loss = epoch_loss / max(1, n_batches) | |
| avg_train_acc = epoch_acc / max(1, n_batches) | |
| print(f"Epoch {epoch+1} - Train Loss: {avg_train_loss:.4f}, Acc: {avg_train_acc:.3f}") | |
| # === Validation === | |
| if (epoch + 1) % config.training.val_every_n_epochs == 0: | |
| model.eval() | |
| val_loss = 0 | |
| val_acc = 0 | |
| n_val = 0 | |
| with torch.no_grad(): | |
| for batch in val_loader: | |
| lip_frames = batch["lip_frames"].to(device) | |
| mel = batch["mel"].to(device) | |
| with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp): | |
| output = model(lip_frames, mel) | |
| loss = SyncNet.contrastive_loss( | |
| output["video_embedding"], | |
| output["audio_embedding"], | |
| model.logit_scale, | |
| ) | |
| sim = output["video_embedding"] @ output["audio_embedding"].t() | |
| preds = sim.argmax(dim=1) | |
| targets = torch.arange(len(preds), device=device) | |
| acc = (preds == targets).float().mean().item() | |
| val_loss += loss.item() | |
| val_acc += acc | |
| n_val += 1 | |
| avg_val_loss = val_loss / max(1, n_val) | |
| avg_val_acc = val_acc / max(1, n_val) | |
| print(f" Val Loss: {avg_val_loss:.4f}, Val Acc: {avg_val_acc:.3f}") | |
| writer.add_scalar("val/loss", avg_val_loss, global_step) | |
| writer.add_scalar("val/accuracy", avg_val_acc, global_step) | |
| # Save best | |
| if avg_val_loss < best_val_loss: | |
| best_val_loss = avg_val_loss | |
| torch.save( | |
| { | |
| "epoch": epoch, | |
| "model_state_dict": model.state_dict(), | |
| "optimizer_state_dict": optimizer.state_dict(), | |
| "val_loss": avg_val_loss, | |
| }, | |
| checkpoint_dir / "syncnet_best.pt", | |
| ) | |
| print(f" Saved best model (val_loss={avg_val_loss:.4f})") | |
| # Save periodic checkpoint | |
| if (epoch + 1) % config.training.save_every_n_epochs == 0: | |
| torch.save( | |
| { | |
| "epoch": epoch, | |
| "model_state_dict": model.state_dict(), | |
| "optimizer_state_dict": optimizer.state_dict(), | |
| }, | |
| checkpoint_dir / f"syncnet_epoch_{epoch+1}.pt", | |
| ) | |
| writer.close() | |
| print(f"\nSyncNet training complete. Best val loss: {best_val_loss:.4f}") | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Train SyncNet") | |
| parser.add_argument("--config", type=str, default="configs/syncnet.yaml") | |
| parser.add_argument("--resume", type=str, default=None, | |
| help="Path to checkpoint to resume from (e.g. checkpoints/syncnet_best.pt)") | |
| args = parser.parse_args() | |
| config = OmegaConf.load(args.config) | |
| train_syncnet(config, resume_checkpoint=args.resume) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 11.1 kB
- Xet hash:
- 7d81c565ae3d456e46697146acde1c8eac5bd9496749a7410c2927f69ee1cacf
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.