configuration_bestrq_conformer.py

from transformers.configuration_utils import PretrainedConfig
from typing import List


class MeralionBestRqConformerConfig(PretrainedConfig):
    """
    This is the configuration class to store the configuration of a [`MeralionBestRqConformer`]. It is used to
    instantiate a BEST-RQ Conformer model according to the specified arguments, defining the model architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        input_dim (`int`, *optional*, defaults to 80):
            The number of input features in the mel-frequency spectrogram.
        input_channels (`int`, *optional*, defaults to 1):
            The number of input channels of the convolutional subsampling layers.
        num_attention_heads (`int`, *optional*, defaults to 8):
            Number of attention heads for each attention layer in the Transformer encoder.
        hidden_size (`int`, *optional*, defaults to 1024):
            Dimensionality of the encoder layers and the pooler layer.
        ffn_dim (`int`, *optional*, defaults to 4096):
            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
        num_hidden_layers (`int`, *optional*, defaults to 24):
            Number of hidden layers in the Transformer encoder.
        conv_depthwise_kernel_size (`int`, *optional*, defaults to 5):
            Kernel size of the depthwise convolution in the Conformer convolution module.
        feat_proj_dropout (`float`, *optional*, defaults to 0.0):
            The dropout probability for the input projection layer.
        activation_dropout (`float`, *optional*, defaults to 0.0):
            The dropout probability for the activation functions in the feed-forward layers.
        hidden_dropout (`float`, *optional*, defaults to 0.0):
            The dropout probability for the hidden layers.
        max_source_positions (`int`, *optional*, defaults to 3000):
            The maximum sequence length that this model might ever be used with.
        no_scale_embedding (`bool`, *optional*, defaults to `False`):
            Whether to scale the embeddings by the square root of the hidden size.
        hidden_act (`str`, *optional*, defaults to `"swish"`):
            The non-linear activation function (function or string) in the encoder and pooler.
        conformer_conv_dropout (`float`, *optional*, defaults to 0.0):
            The dropout probability for the Conformer convolution module.
        position_embeddings_type (`str`, *optional*, defaults to `"relative"`):
            The type of position embeddings to use. Can be `"relative"` or `"rotary"`.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout probability for the attention layers.
        rotary_embedding_base (`int`, *optional*, defaults to 10000):
            The base for the rotary position embeddings.
        layerdrop (`float`, *optional*, defaults to 0.0):
            The LayerDrop probability.
        self_condition_layers (`List`, *optional*, defaults to `[]`):
            A list of layer indices where self-conditioning should be applied.
        use_weighted_sum (`bool`, *optional*, defaults to `True`):
            Whether to use a weighted sum of all hidden states for the final output of the LSTM-CTC model.
        lstm_dim (`int`, *optional*, defaults to 768):
            The hidden size of the LSTM layers in the LSTM-CTC head.
        lstm_num_layers (`int`, *optional*, defaults to 2):
            The number of layers in the LSTM of the LSTM-CTC head.
        lstm_dropout_prob (`float`, *optional*, defaults to 0.0):
            The dropout probability for the LSTM layers in the LSTM-CTC head.
        final_dropout (`float`, *optional*, defaults to 0.0):
            The dropout probability for the final layer before the CTC loss.
        vocab_size (`int`, *optional*):
            Vocabulary size of the model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`MeralionBestRqModelForCTC`].
        ctc_loss_reduction (`str`, *optional*, defaults to `"sum"`):
            The reduction to apply to the output of `torch.nn.functional.ctc_loss`.
        ctc_zero_infinity (`bool`, *optional*, defaults to `False`):
            Whether to zero infinite losses and gradients in `torch.nn.functional.ctc_loss`.
    """
    model_type = "meralion_bestrq"

    def __init__(
        self,
        input_dim: int = 80,
        input_channels: int = 1,
        num_attention_heads: int = 8,
        hidden_size: int = 1024, #embed_dim
        ffn_dim: int = 4096,
        num_hidden_layers: int = 24,
        conv_depthwise_kernel_size: int = 5,
        feat_proj_dropout: float = 0., #for input_projection
        activation_dropout: float = 0.,
        hidden_dropout: float = 0.,
        max_source_positions: int = 3000,
        no_scale_embedding: bool = False,
        hidden_act: str = "swish",
        conformer_conv_dropout: float = 0.,
        position_embeddings_type: str = "relative",
        attention_dropout: float = 0.,
        rotary_embedding_base: int = 10000,
        layerdrop = 0.,
        self_condition_layers: List = [], # asr
        use_weighted_sum: bool = True, #lstm
        lstm_dim: int = 768, #lstm
        lstm_num_layers: int = 2, #lstm
        lstm_dropout_prob = 0., #lstm
        final_dropout = 0., #ctc
        vocab_size = None, #ctc
        ctc_loss_reduction = 'sum', #ctc
        ctc_zero_infinity = False, #ctc
        **kwargs,
    ):

        self.input_dim = input_dim
        self.input_channels = input_channels
        self.num_attention_heads = num_attention_heads
        self.hidden_size = hidden_size
        self.ffn_dim = ffn_dim
        self.num_hidden_layers = num_hidden_layers
        self.conv_depthwise_kernel_size = conv_depthwise_kernel_size
        self.feat_proj_dropout = feat_proj_dropout
        self.activation_dropout = activation_dropout
        self.hidden_dropout = hidden_dropout
        self.max_source_positions = max_source_positions
        self.no_scale_embedding = no_scale_embedding
        self.hidden_act = hidden_act
        self.conformer_conv_dropout = conformer_conv_dropout
        self.position_embeddings_type = position_embeddings_type
        self.attention_dropout = attention_dropout
        self.rotary_embedding_base = rotary_embedding_base
        self.layerdrop = layerdrop
        self.self_condition_layers = self_condition_layers
        self.use_weighted_sum = use_weighted_sum
        self.lstm_dim = lstm_dim
        self.lstm_num_layers = lstm_num_layers
        self.lstm_dropout_prob = lstm_dropout_prob
        self.final_dropout = final_dropout
        self.vocab_size = vocab_size
        self.ctc_loss_reduction = ctc_loss_reduction
        self.ctc_zero_infinity = ctc_zero_infinity

        self.inputs_to_logits_ratio = 640
        super().__init__(**kwargs)