avh-align_core/train.py

import os
import torch
from torch.utils.data import DataLoader
from tqdm import tqdm
from pathlib import Path
import socket

from config import get_args
from dataset import FeatureDataset
from model import FusionModel
from utils import print_args

def save_checkpoint(state, epoch, is_best, save_path, model_name):
    """Save model checkpoint"""
    best_model_path = os.path.join(save_path, f'{model_name}.pt')
    
    if is_best:
        dir_path = os.path.dirname(best_model_path)
        os.makedirs(dir_path, exist_ok=True)

        torch.save(state, best_model_path)

def run_epoch(dataloader, model, tau, penalty_coefficient, optimizer=None, is_training=True, use_tqdm=False, 
              intermediate_logging=False, log_interval=500):
    """Runs a single epoch for training or validation."""
    if is_training:
        model.train()
    else:
        model.eval()

    total_loss = 0
    total_samples = 0
    logsoftmax = torch.nn.LogSoftmax(dim=1)

    with torch.set_grad_enabled(is_training):
        loader = tqdm(dataloader) if use_tqdm else dataloader
        for idx, batch in enumerate(loader):
            visual_frame, audio_window, video_name, video_frames = batch
            current_batch_size = visual_frame.size()[0]

            visual_frame = visual_frame.to(model.device)
            audio_window = audio_window.to(model.device)

            # Repeat video frame to match audio frames (2*tau+1 times)
            visual_central_frame = visual_frame.unsqueeze(1).repeat(1, 2 * tau + 1, 1)

            outputs = model(visual_central_frame, audio_window)
            outputs = outputs.squeeze()
        
            synchronization_scores = logsoftmax(outputs)[:, tau]
            loss = -torch.sum(synchronization_scores)

            total_loss += loss.item()
            total_samples += current_batch_size

            if is_training:
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

                if intermediate_logging and idx % log_interval == 0 and idx > 0:
                    avg_loss = total_loss / total_samples
                    print(f"Step [{idx}/{len(loader)}] \t Loss: {avg_loss:.6f} \t Output mean score this batch: {torch.mean(outputs).item():.3f} \t sync_score avg: {torch.mean(synchronization_scores).item():.3f}")
            
    avg_loss = total_loss / total_samples if total_samples > 0 else 0
    return avg_loss

def main():
    num_workers = 32
    args = get_args()
    print_args(args)
    
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    
    train_dataset = FeatureDataset(
        os.path.join(args.metadata_root_path, "train_metadata.csv"), 
        os.path.join(args.data_root_path, "train"),
        tau=args.tau,
    )
    val_dataset = FeatureDataset(
        os.path.join(args.metadata_root_path, "val_metadata.csv"), 
        os.path.join(args.data_root_path, "train"),
        tau=args.tau,
    )
        
    train_loader = DataLoader(
        train_dataset, 
        batch_size=args.batch_size,
        num_workers=num_workers,
        pin_memory=True
    )
    val_loader = DataLoader(
        val_dataset,
        batch_size=args.batch_size,
        num_workers=num_workers,
        pin_memory=True
    )

    print(f"Train dataset size: {len(train_dataset)}")
    print(f"Val dataset size: {len(val_dataset)}")
    
    model = FusionModel().to(device)
    model.device = device
    
    # Initialize optimizer and scheduler
    optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
        optimizer,
        mode='min',
        factor=0.1,
        patience=args.scheduler_patience
    )

    # Training loop
    best_val_loss = float('inf')
    epochs_without_improvement = 0
    previous_epoch_lr = args.learning_rate

    for epoch in range(args.epochs):
        print(f"Epoch {epoch + 1}/{args.epochs}")

        # Train
        train_loss = run_epoch(
            train_loader, model, args.tau, args.penalty_coefficient, optimizer,
            is_training=True,
            use_tqdm=args.use_tqdm,
            intermediate_logging=not args.no_intermediate_logging,
            log_interval=args.log_interval
        )
        print(f"Training - Loss: {train_loss:.6f}")

        # Validation
        val_loss = run_epoch(
            val_loader, model, args.tau, args.penalty_coefficient,
            is_training=False,
            use_tqdm=args.use_tqdm
        )
        print(f"Validation - Loss: {val_loss:.6f}")

        # Scheduler step
        scheduler.step(val_loss)
        current_lr = scheduler.get_last_lr()[-1]
        if current_lr != previous_epoch_lr:
            print(f"Learning rate has changed to {current_lr}")
            previous_epoch_lr = current_lr

        # Save checkpoint
        is_best = val_loss < best_val_loss
        checkpoint = {
            'epoch': epoch + 1,
            'state_dict': model.state_dict(),
            'optimizer': optimizer.state_dict(),
            'scheduler': scheduler.state_dict(),
            'best_val_loss': best_val_loss,
            'args': args
        }
        
        model_name = args.name
        save_checkpoint(checkpoint, epoch, is_best, args.save_path, model_name)

        if is_best:
            best_val_loss = val_loss
            print(f"New best model saved with validation loss: {val_loss:.6f}")
            epochs_without_improvement = 0
        else:
            epochs_without_improvement += 1
            print(f"Validation loss did not improve for {epochs_without_improvement} epoch(s)")

        # Early stopping
        if epochs_without_improvement >= args.early_stopping_patience:
            print(f"Early stopping triggered after {epochs_without_improvement} epochs without improvement")
            break

    print("Training finished")

if __name__ == "__main__":
    main()