models/helpers/tools.py

from typing import List, Tuple

import numpy as np
import torch
import torch.nn.functional as F


def get_device() -> torch.device:
    r"""Function returns the device where the model and tensors should be placed.

    Returns
        torch.device: The device where the model and tensors should be placed.
    """
    return torch.device("cuda" if torch.cuda.is_available() else "cpu")


def pad(input_ele: List[torch.Tensor], max_len: int) -> torch.Tensor:
    r"""Takes a list of 1D or 2D tensors and pads them to match the maximum length.

    Args:
        input_ele (List[torch.Tensor]): The list of tensors to be padded.
        max_len (int): The length to which the tensors should be padded.

    Returns:
        torch.Tensor: A tensor containing all the padded input tensors.
    """
    # Create an empty list to store the padded tensors
    out_list = torch.jit.annotate(List[torch.Tensor], [])
    for batch in input_ele:
        if len(batch.shape) == 1:
            # Perform padding for 1D tensor
            one_batch_padded = F.pad(
                batch,
                (0, max_len - batch.size(0)),
                "constant",
                0.0,
            )
        else:
            # Perform padding for 2D tensor
            one_batch_padded = F.pad(
                batch,
                (0, 0, 0, max_len - batch.size(0)),
                "constant",
                0.0,
            )
        # Append the padded tensor to the list
        out_list.append(one_batch_padded)

    # Stack all the tensors in the list into a single tensor
    return torch.stack(out_list)


def get_mask_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
    r"""Generate a mask tensor from a tensor of sequence lengths.

    Args:
        lengths (torch.Tensor): A tensor of sequence lengths of shape: (batch_size, )

    Returns:
        torch.Tensor: A mask tensor of shape: (batch_size, max_len) where max_len is the
            maximum sequence length in the provided tensor. The mask tensor has a value of
            True at each position that is more than the length of the sequence (padding positions).

    Example:
      lengths: `torch.tensor([2, 3, 1, 4])`
      Mask tensor will be: `torch.tensor([
            [False, False, True, True],
            [False, False, False, True],
            [False, True, True, True],
            [False, False, False, False]
        ])`
    """
    # Get batch size
    batch_size = lengths.shape[0]

    # Get maximum sequence length in the batch
    max_len = int(torch.max(lengths).item())

    # Generate a tensor of shape (batch_size, max_len)
    # where each row contains values from 0 to max_len
    ids = (
        torch.arange(0, max_len, device=lengths.device)
        .unsqueeze(0)
        .expand(batch_size, -1)
    )
    # Compare each value in the ids tensor with
    # corresponding sequence length to generate a mask.
    # The mask will have True at positions where id >= sequence length,
    # indicating padding positions in the original sequences
    return ids >= lengths.unsqueeze(1).type(torch.int64).expand(-1, max_len)


def stride_lens_downsampling(lens: torch.Tensor, stride: int = 2) -> torch.Tensor:
    r"""Function computes the lengths of 1D tensor when applying a stride for downsampling.

    Args:
        lens (torch.Tensor): Tensor containing the lengths to be downsampled.
        stride (int, optional): The stride to be used for downsampling. Defaults to 2.

    Returns:
        torch.Tensor: A tensor of the same shape as the input containing the downsampled lengths.
    """
    # The torch.ceil function is used to handle cases where the length is not evenly divisible
    # by the stride. The torch.ceil function rounds up to the nearest integer, ensuring that
    # each item is present at least once in the downsampled lengths.
    # Finally, the .int() is used to convert the resulting float32 tensor to an integer tensor.
    return torch.ceil(lens / stride).int()


def calc_same_padding(kernel_size: int) -> Tuple[int, int]:
    r"""Calculates the necessary padding for 'same' padding in convolutional operations.

    For 'same' padding, the output size is the same as the input size for `stride=1`. This function returns
    two integers, representing the padding to be added on either side of the input to achieve 'same' padding.

    Args:
        kernel_size (int): Size of the convolving kernel.

    Returns:
        Tuple[int, int]: A tuple of two integers representing the number of padding elements to be applied on
        left and right (or top and bottom for 2D) of the input tensor respectively.
    """
    # Check if kernel_size is an integer greater than zero
    if not isinstance(kernel_size, int) or kernel_size <= 0:
        raise ValueError("kernel_size must be an integer greater than zero")

    # Determine base padding amount (equal to half the kernel size, truncated down)
    pad = kernel_size // 2

    # Return padding for each side of the kernel. If kernel size is odd, padding is (pad, pad).
    # If kernel size is even, padding is (pad, pad - 1) because we can't pad equally on both sides.
    return (pad, pad - (kernel_size + 1) % 2)


def initialize_embeddings(shape: Tuple[int, ...]) -> torch.Tensor:
    r"""Initialize embeddings using Kaiming initialization (He initialization).

    This method is specifically designed for 2D matrices and helps to avoid
    the vanishing/exploding gradient problem in deep neural networks.
    This is achieved by keeping the variance of the outputs of a layer to be
    the same as the variance of its inputs.

    Args:
        shape (Tuple[int, ...]): The shape of the embedding matrix to create, denoted as a tuple of integers.
                                 The shape should comprise 2 dimensions, i.e., (embedding_dim, num_embeddings).

    Raises:
        AssertionError: if the provided shape is not 2D.

    Returns:
        torch.Tensor: the created embedding matrix.
    """
    # Check if the input shape is 2D
    assert len(shape) == 2, "Can only initialize 2-D embedding matrices ..."

    # Initialize the embedding matrix using Kaiming initialization
    return torch.randn(shape) * np.sqrt(2 / shape[1])