metrics.py

import torch
from torchmetrics import Metric

class MyAccuracy(Metric):
    """
    Accuracy metric costomized for handling sequences with padding.

    Methods:
        update(self, logits, labels, num_labels): Update the accuracy based on 
        model predictions and ground truth labels.

        compute(self): Compute the accuracy.

    Attributes:
        total (torch.Tensor): Total number of non-padding elements.
        correct (torch.Tensor): Number of correctly predicted non-padding elements.
    """
    def __init__(self):
        super().__init__()
        self.add_state('total', default=torch.tensor(0), dist_reduce_fx='sum')
        self.add_state('correct', default=torch.tensor(0), dist_reduce_fx='sum')

    def update(self, logits: torch.Tensor, labels: torch.Tensor, num_labels: int) -> None:
        """
        Args:
            logits (torch.Tensor): Model predictions.
            labels (torch.Tensor): Ground truth labels.
            num_labels (int): Number of unique labels.
        """
        flattened_targets = labels.view(-1) # shape (batch_size, sequence_len)
        active_logits = logits.view(-1, num_labels) # shape (batch_size * sequence_len, num_labels)
        flattened_predictions = torch.argmax(active_logits, axis=1) # shape (batch_size * sequence_len)

        # compute accuracy only at active labels
        active_accuracy = labels.view(-1) != -100 # shape (batch_size, sequnce_len)
        ac_labels = torch.masked_select(flattened_targets, active_accuracy)
        predictions = torch.masked_select(flattened_predictions, active_accuracy)

        self.correct += torch.sum(ac_labels == predictions)
        self.total += torch.numel(ac_labels)

    def compute(self) -> torch.Tensor:
        """
        Calculate the accuracy.
        """
        return self.correct.float() / self.total.float()