src/validation.py

from __future__ import annotations

import torch
from torch.utils.data import DataLoader

from src.losses import compute_loss_for_phase
from src.training_data import move_batch_to_device

def evaluate_validation_loss(
    phase: str,
    model,
    dataloader: DataLoader,
    device: torch.device,
    alpha: float,
    temperature: float,
    online_kd_token_chunk_size: int = 2048,
    teacher_model=None,
    max_batches: int = -1,
) -> dict[str, float | int]:
    was_training = model.training
    model.eval()

    total_loss = 0.0
    total_ce = 0.0
    total_kd = 0.0
    batches = 0

    with torch.inference_mode():
        for batch in dataloader:
            if max_batches > 0 and batches >= max_batches:
                break
            batch = move_batch_to_device(batch, device)
            input_ids = batch["input_ids"]
            attention_mask = batch["attention_mask"]
            labels = batch["labels"]
            loss_mask = batch["loss_mask"]

            logits = model(input_ids=input_ids, attention_mask=attention_mask).logits

            if phase == "online_kd" and teacher_model is not None:
                teacher_logits = teacher_model(input_ids=input_ids, attention_mask=attention_mask).logits
            else:
                teacher_logits = None

            loss, ce, kd = compute_loss_for_phase(
                phase,
                logits,
                labels,
                loss_mask,
                batch,
                alpha,
                temperature,
                teacher_logits=teacher_logits,
                online_kd_token_chunk_size=online_kd_token_chunk_size,
            )
            total_loss += float(loss.detach().item())
            total_ce += float(ce.detach().item())
            total_kd += float(kd.detach().item())
            batches += 1

    if was_training:
        model.train()

    denom = max(batches, 1)
    return {
        "loss": total_loss / denom,
        "ce": total_ce / denom,
        "kd": total_kd / denom,
        "batches": batches,
    }