minhluanquach's picture
download
raw
22 kB
"""
Main LipSync Model Training Script
=====================================
Full training pipeline with:
- Mixed precision (FP16/BF16)
- Gradient checkpointing & accumulation
- EMA (Exponential Moving Average)
- TensorBoard logging
- Checkpoint saving & resume
- Validation with visual samples
Usage:
python -m src.training.train_model --config configs/train.yaml
python -m src.training.train_model --config configs/test.yaml # Quick test
"""
import argparse
import copy
import gc
import os
import tempfile
import time
from pathlib import Path
import torch
import torch.nn as nn
import torch.nn.functional as F
from omegaconf import OmegaConf
from torch.amp import autocast, GradScaler
from torch.utils.data import DataLoader, random_split
from torch.utils.tensorboard import SummaryWriter
from torchvision.utils import make_grid, save_image
from tqdm import tqdm
from src.data.dataset import LipSyncDataset
from src.models.unet import LipSyncUNet
from src.training.losses import CompositeLoss
class EMA:
"""Exponential Moving Average for model parameters.
Maintains a shadow copy of model weights that are updated as:
shadow = decay * shadow + (1 - decay) * current
EMA weights typically produce smoother and better results at inference.
"""
def __init__(self, model: nn.Module, decay: float = 0.9999, update_every: int = 10):
self.decay = decay
self.update_every = update_every
self.step = 0
self.shadow = {}
self.backup = {}
for name, param in model.named_parameters():
if param.requires_grad:
self.shadow[name] = param.data.clone()
@torch.no_grad()
def update(self, model: nn.Module):
self.step += 1
if self.step % self.update_every != 0:
return
for name, param in model.named_parameters():
if param.requires_grad and name in self.shadow:
self.shadow[name].lerp_(param.data, 1 - self.decay)
def apply(self, model: nn.Module):
"""Apply EMA weights to model (for evaluation)."""
for name, param in model.named_parameters():
if param.requires_grad and name in self.shadow:
self.backup[name] = param.data.clone()
param.data.copy_(self.shadow[name])
def restore(self, model: nn.Module):
"""Restore original weights after evaluation."""
for name, param in model.named_parameters():
if param.requires_grad and name in self.backup:
param.data.copy_(self.backup[name])
self.backup = {}
def state_dict(self):
return {"shadow": self.shadow, "step": self.step, "decay": self.decay}
def load_state_dict(self, state_dict):
self.shadow = state_dict["shadow"]
self.step = state_dict["step"]
self.decay = state_dict["decay"]
def create_lr_scheduler(optimizer, config, total_steps):
"""Create learning rate scheduler with warmup."""
warmup_steps = config.training.scheduler.warmup_steps
min_lr = config.training.scheduler.min_lr
base_lr = config.training.learning_rate
def lr_lambda(step):
if step < warmup_steps:
return step / max(1, warmup_steps)
progress = (step - warmup_steps) / max(1, total_steps - warmup_steps)
return max(min_lr / base_lr, 0.5 * (1 + torch.cos(torch.tensor(progress * 3.14159)).item()))
return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda)
def save_checkpoint(
path: Path,
epoch: int,
global_step: int,
model: nn.Module,
optimizer,
scheduler,
scaler,
ema,
best_val_loss: float,
config,
patience_counter: int = 0,
):
"""Save training checkpoint."""
state = {
"epoch": epoch,
"global_step": global_step,
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"scheduler_state_dict": scheduler.state_dict(),
"scaler_state_dict": scaler.state_dict() if scaler.is_enabled() else None,
"best_val_loss": best_val_loss,
"patience_counter": patience_counter,
"config": OmegaConf.to_container(config),
}
if ema is not None:
state["ema_state_dict"] = ema.state_dict()
# Atomic save: write to temp file then rename to prevent corruption
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
try:
os.close(fd)
torch.save(state, tmp_path)
os.replace(tmp_path, str(path))
except BaseException:
if os.path.exists(tmp_path):
os.remove(tmp_path)
raise
def load_checkpoint(path: Path, model, optimizer, scheduler, scaler, ema, device):
"""Load training checkpoint."""
ckpt = torch.load(path, map_location=device, weights_only=False)
model.load_state_dict(ckpt["model_state_dict"])
optimizer.load_state_dict(ckpt["optimizer_state_dict"])
scheduler.load_state_dict(ckpt["scheduler_state_dict"])
if scaler.is_enabled() and ckpt.get("scaler_state_dict"):
scaler.load_state_dict(ckpt["scaler_state_dict"])
if ema is not None and "ema_state_dict" in ckpt:
ema.load_state_dict(ckpt["ema_state_dict"])
return ckpt["epoch"], ckpt["global_step"], ckpt.get("best_val_loss", float("inf")), ckpt.get("patience_counter", 0)
def validate(
model: nn.Module,
val_loader: DataLoader,
loss_fn: CompositeLoss,
device: torch.device,
amp_dtype: torch.dtype,
use_amp: bool,
writer: SummaryWriter,
global_step: int,
num_samples: int = 4,
vae=None,
):
"""Run validation and log visual samples."""
model.eval()
total_loss = 0
n_batches = 0
sample_preds = []
sample_targets = []
sample_masked = []
with torch.no_grad():
for batch in val_loader:
target_latent = batch["target_latent"].to(device)
ref_latent = batch["reference_latent"].to(device)
audio_features = batch["audio_features"].to(device)
B, C, H, W = target_latent.shape
# Create mask
mask = torch.zeros(B, 1, H, W, device=device)
mask[:, :, H // 2 :, :] = 1.0
masked_latent = target_latent * (1 - mask)
with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp):
pred_latent = model(
masked_latent=masked_latent,
reference_latent=ref_latent,
mask=mask,
audio_features=audio_features,
)
losses = loss_fn(pred_latent, target_latent, mask)
total_loss += losses["total"].item()
n_batches += 1
# Collect samples for visualization
if len(sample_preds) < num_samples:
sample_preds.append(pred_latent[:1])
sample_targets.append(target_latent[:1])
sample_masked.append(masked_latent[:1])
avg_loss = total_loss / max(1, n_batches)
writer.add_scalar("val/loss", avg_loss, global_step)
# Decode and visualize samples if VAE is available
if vae is not None and sample_preds:
try:
pred_latents = torch.cat(sample_preds[:num_samples], dim=0)
target_latents = torch.cat(sample_targets[:num_samples], dim=0)
masked_latents = torch.cat(sample_masked[:num_samples], dim=0)
scale = vae.config.scaling_factor
with torch.no_grad():
pred_imgs = vae.decode(pred_latents / scale).sample
target_imgs = vae.decode(target_latents / scale).sample
masked_imgs = vae.decode(masked_latents / scale).sample
# Create comparison grid: masked | predicted | ground truth
comparison = torch.cat([masked_imgs, pred_imgs, target_imgs], dim=0)
grid = make_grid(
comparison.clamp(-1, 1) * 0.5 + 0.5, # Scale to [0, 1]
nrow=len(sample_preds),
padding=2,
)
writer.add_image("val/comparison", grid, global_step)
except Exception as e:
print(f"Warning: Could not generate validation images: {e}")
model.train()
return avg_loss
def train(config):
"""Main training loop."""
device = torch.device(config.hardware.device if torch.cuda.is_available() else "cpu")
print(f"Device: {device}")
if torch.cuda.is_available():
print(f"GPU: {torch.cuda.get_device_name()}")
print(f"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB")
# === Dataset ===
dataset = LipSyncDataset(
processed_dir=config.data.processed_dir,
cache_dir=config.data.cache_dir,
image_size=config.data.image_size,
fps=config.data.fps,
mel_step_size=config.data.mel_step_size,
num_audio_frames=config.data.num_audio_frames,
sample_rate=config.data.sample_rate,
)
if len(dataset) == 0:
print("ERROR: Dataset is empty. Run preprocessing and feature extraction first.")
return
# Train/val split
val_size = max(1, int(len(dataset) * 0.05))
train_size = len(dataset) - val_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
train_loader = DataLoader(
train_dataset,
batch_size=config.training.batch_size,
shuffle=True,
num_workers=config.data.num_workers,
pin_memory=config.data.pin_memory,
drop_last=True,
)
val_loader = DataLoader(
val_dataset,
batch_size=config.training.batch_size,
shuffle=False,
num_workers=config.data.num_workers,
pin_memory=config.data.pin_memory,
)
print(f"Train: {len(train_dataset)}, Val: {len(val_dataset)}")
# === Model (UNet only - frozen components loaded separately) ===
unet_cfg = OmegaConf.to_container(config.model.unet)
model = LipSyncUNet(**unet_cfg).to(device)
param_info = model.get_param_count()
print(f"UNet parameters: {param_info['trainable_M']:.2f}M trainable")
# === Load frozen VAE (for validation visualization) ===
vae = None
try:
from diffusers import AutoencoderKL
vae = AutoencoderKL.from_pretrained(
config.model.vae_model_id, subfolder=config.model.vae_subfolder
).to(device)
vae.eval()
vae.requires_grad_(False)
print("VAE loaded for loss computation + visualization")
except Exception as e:
print(f"VAE not loaded (perceptual/sync/GAN losses disabled): {e}")
# === Loss function ===
syncnet = None
syncnet_path = config.model.syncnet.get("checkpoint") if hasattr(config.model, "syncnet") else None
if syncnet_path and Path(syncnet_path).exists():
from src.models.syncnet import SyncNet
syncnet = SyncNet().to(device)
ckpt = torch.load(syncnet_path, map_location=device, weights_only=True)
syncnet.load_state_dict(ckpt["model_state_dict"])
syncnet.eval()
syncnet.requires_grad_(False)
print(f"SyncNet loaded from {syncnet_path}")
loss_fn = CompositeLoss(
weight_l1=config.training.loss_weights.l1,
weight_perceptual=config.training.loss_weights.perceptual,
weight_sync=config.training.loss_weights.sync,
weight_gan=config.training.loss_weights.gan,
syncnet=syncnet,
).to(device)
# === Optimizer ===
optimizer = torch.optim.AdamW(
model.parameters(),
lr=config.training.learning_rate,
weight_decay=config.training.weight_decay,
betas=tuple(config.training.betas),
)
# Scheduler
steps_per_epoch = len(train_loader) // config.training.gradient_accumulation_steps
total_steps = steps_per_epoch * config.training.max_epochs
scheduler = create_lr_scheduler(optimizer, config, total_steps)
# Mixed precision
use_amp = config.training.mixed_precision in ("fp16", "bf16")
scaler = GradScaler(enabled=use_amp and config.training.mixed_precision == "fp16")
amp_dtype = torch.bfloat16 if config.training.mixed_precision == "bf16" else torch.float16
# EMA
ema = None
if config.training.ema.get("enabled", False):
ema = EMA(
model,
decay=config.training.ema.decay,
update_every=config.training.ema.update_every,
)
print(f"EMA enabled (decay={config.training.ema.decay})")
# Logging
log_dir = Path(config.training.log_dir)
log_dir.mkdir(parents=True, exist_ok=True)
writer = SummaryWriter(log_dir=str(log_dir))
checkpoint_dir = Path(config.training.checkpoint_dir)
checkpoint_dir.mkdir(parents=True, exist_ok=True)
# Resume
start_epoch = 0
global_step = 0
best_val_loss = float("inf")
patience_counter = 0
patience = config.training.get("early_stopping_patience", 0) # 0 = disabled
if config.training.resume_from and Path(config.training.resume_from).exists():
print(f"Resuming from {config.training.resume_from}")
start_epoch, global_step, best_val_loss, patience_counter = load_checkpoint(
Path(config.training.resume_from), model, optimizer, scheduler, scaler, ema, device
)
print(f"Resumed at epoch {start_epoch}, step {global_step}, patience={patience_counter}/{patience}")
# === Training Loop ===
accum_steps = config.training.gradient_accumulation_steps
for epoch in range(start_epoch, config.training.max_epochs):
model.train()
epoch_loss = 0
epoch_losses = {}
n_batches = 0
pbar = tqdm(
train_loader,
desc=f"Epoch {epoch+1}/{config.training.max_epochs}",
)
optimizer.zero_grad()
for batch_idx, batch in enumerate(pbar):
target_latent = batch["target_latent"].to(device)
ref_latent = batch["reference_latent"].to(device)
audio_features = batch["audio_features"].to(device)
target_image = batch["target_image"].to(device)
mel_segment = batch["mel_segment"].to(device)
lip_frames = batch["lip_frames"].to(device)
B, C, H, W = target_latent.shape
# Create mask
mask = torch.zeros(B, 1, H, W, device=device)
mask[:, :, H // 2 :, :] = 1.0
masked_latent = target_latent * (1 - mask)
# Forward pass with mixed precision
with autocast(device_type="cuda", dtype=amp_dtype, enabled=use_amp):
pred_latent = model(
masked_latent=masked_latent,
reference_latent=ref_latent,
mask=mask,
audio_features=audio_features,
)
# Decode predicted latents to pixel space for perceptual/sync/GAN losses
pred_image = None
pred_lip_frames = None
if vae is not None:
with torch.no_grad():
scale = vae.config.scaling_factor
pred_image = vae.decode(pred_latent / scale).sample
# Resize to match target_image if resolutions differ
target_h = target_image.shape[2]
if pred_image.shape[2] != target_h:
pred_image = F.interpolate(
pred_image, size=(target_h, target_h),
mode="bilinear", align_corners=False,
)
# Crop lip region from predicted image for SyncNet loss
img_h = pred_image.shape[2]
lip_crop = pred_image[:, :, img_h // 2:, :] # Lower half
lip_crop = F.interpolate(lip_crop, size=(96, 96), mode="bilinear", align_corners=False)
# SyncNet expects (B, T, C, H, W) — use single predicted frame as center
T = lip_frames.shape[1]
pred_lip_frames = lip_crop.unsqueeze(1).expand(-1, T, -1, -1, -1)
# Compute losses (perceptual/sync/GAN activate when images are available)
losses = loss_fn(
pred_latent=pred_latent,
target_latent=target_latent,
mask=mask,
pred_image=pred_image,
target_image=target_image,
lip_frames=pred_lip_frames,
mel_segment=mel_segment,
)
loss = losses["total"] / accum_steps
# Backward
if scaler.is_enabled():
scaler.scale(loss).backward()
else:
loss.backward()
# Accumulate gradients
if (batch_idx + 1) % accum_steps == 0:
if scaler.is_enabled():
scaler.unscale_(optimizer)
nn.utils.clip_grad_norm_(model.parameters(), 1.0)
if scaler.is_enabled():
scaler.step(optimizer)
scaler.update()
else:
optimizer.step()
optimizer.zero_grad()
scheduler.step()
if ema is not None:
ema.update(model)
global_step += 1
# Track losses
epoch_loss += losses["total"].item()
for k, v in losses.items():
if k != "total":
epoch_losses[k] = epoch_losses.get(k, 0) + v.item()
n_batches += 1
# Progress bar
pbar.set_postfix(
loss=f"{losses['total'].item():.4f}",
lr=f"{scheduler.get_last_lr()[0]:.2e}",
)
# Logging
if global_step > 0 and global_step % config.training.log_every_n_steps == 0:
writer.add_scalar("train/loss", losses["total"].item(), global_step)
writer.add_scalar("train/lr", scheduler.get_last_lr()[0], global_step)
for k, v in losses.items():
if k != "total":
writer.add_scalar(f"train/{k}", v.item(), global_step)
avg_loss = epoch_loss / max(1, n_batches)
print(f"Epoch {epoch+1} - Loss: {avg_loss:.4f}")
for k, v in epoch_losses.items():
print(f" {k}: {v / max(1, n_batches):.4f}")
# === Validation ===
if (epoch + 1) % config.training.val_every_n_epochs == 0:
if ema is not None:
ema.apply(model)
val_loss = validate(
model=model,
val_loader=val_loader,
loss_fn=loss_fn,
device=device,
amp_dtype=amp_dtype,
use_amp=use_amp,
writer=writer,
global_step=global_step,
num_samples=config.training.num_val_samples,
vae=vae,
)
print(f" Val Loss: {val_loss:.4f}")
if ema is not None:
ema.restore(model)
# Save best
if val_loss < best_val_loss:
best_val_loss = val_loss
patience_counter = 0
save_checkpoint(
checkpoint_dir / "best_model.pt",
epoch, global_step, model, optimizer, scheduler, scaler, ema,
best_val_loss, config, patience_counter,
)
print(f" Saved best model (val_loss={val_loss:.4f})")
else:
patience_counter += 1
if patience > 0:
print(f" No improvement ({patience_counter}/{patience})")
if patience_counter >= patience:
print(f" Early stopping triggered after {patience_counter} validations without improvement.")
break
# Periodic checkpoint
if (epoch + 1) % config.training.save_every_n_epochs == 0:
save_checkpoint(
checkpoint_dir / f"checkpoint_epoch_{epoch+1}.pt",
epoch, global_step, model, optimizer, scheduler, scaler, ema,
best_val_loss, config, patience_counter,
)
# Keep only top-k checkpoints
ckpts = sorted(
checkpoint_dir.glob("checkpoint_epoch_*.pt"),
key=lambda p: p.stat().st_mtime,
reverse=True,
)
for old_ckpt in ckpts[config.training.save_top_k:]:
old_ckpt.unlink()
# Save final model
save_checkpoint(
checkpoint_dir / "final_model.pt",
config.training.max_epochs, global_step, model, optimizer, scheduler, scaler, ema,
best_val_loss, config, patience_counter,
)
writer.close()
print(f"\nTraining complete! Best val loss: {best_val_loss:.4f}")
print(f"Checkpoints saved to: {checkpoint_dir}")
def main():
parser = argparse.ArgumentParser(description="Train LipSync Model")
parser.add_argument("--config", type=str, default="configs/train.yaml")
args = parser.parse_args()
config = OmegaConf.load(args.config)
train(config)
if __name__ == "__main__":
main()

Xet Storage Details

Size:
22 kB
·
Xet hash:
a8526e6f712cf3b21948d5ff04e00ff4fa63f34b11910ecaa773891c4dec0c49

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.