| | |
| | |
| | |
| | |
| |
|
| | import copy |
| | import os |
| | from typing import Union |
| |
|
| | from transformers.configuration_utils import PretrainedConfig |
| | from transformers.utils import logging |
| |
|
| | logger = logging.get_logger(__name__) |
| |
|
| |
|
| | class AudioConfig(PretrainedConfig): |
| | r""" |
| | This is the configuration class to store the configuration of an Audio Encoder Model. |
| | It is used to instantiate an audio encoder according to the specified arguments, |
| | defining the model architecture. |
| | |
| | Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. |
| | Read the documentation from [`PretrainedConfig`] for more information. |
| | |
| | Args: |
| | speech_encoder (`str`, *optional*, defaults to `"whisper-base"`): |
| | Path or name of the speech encoder model. |
| | speech_encoder_type (`str`, *optional*, defaults to `"whisper"`): |
| | Type of speech encoder to use. |
| | speech_projector_type (`str`, *optional*, defaults to `"linear"`): |
| | Type of speech projector to use for feature alignment. |
| | speech_encoder_ds_rate (`int`, *optional*, defaults to 5): |
| | Downsampling rate for speech features. |
| | speech_encoder_hidden_size (`int`, *optional*, defaults to 512): |
| | Hidden size of the speech encoder. |
| | mel_bins (`int`, *optional*, defaults to 80): |
| | Number of mel-frequency bins for spectrogram features. |
| | sample_rate (`int`, *optional*, defaults to 16000): |
| | Audio sample rate in Hz. |
| | frame_length (`float`, *optional*, defaults to 25.0): |
| | Frame length in milliseconds for audio processing. |
| | frame_shift (`float`, *optional*, defaults to 10.0): |
| | Frame shift in milliseconds for audio processing. |
| | use_beats (`bool`, *optional*, defaults to False): |
| | Whether to use BEATs model for audio feature extraction. |
| | beats_model_path (`str`, *optional*, defaults to None): |
| | Path to BEATs model if use_beats is True. |
| | whisper_config (`dict`, *optional*, defaults to None): |
| | Configuration dictionary for Whisper model parameters. |
| | """ |
| |
|
| | model_type = 'audio_encoder' |
| |
|
| | def __init__( |
| | self, |
| | speech_encoder="whisper-base", |
| | speech_encoder_type="whisper", |
| | speech_projector_type="linear", |
| | speech_encoder_ds_rate=5, |
| | speech_encoder_hidden_size=1280, |
| | mel_bins=80, |
| | sample_rate=16000, |
| | frame_length=25.0, |
| | frame_shift=10.0, |
| | use_beats=False, |
| | beats_model_path=None, |
| | whisper_config=None, |
| | **kwargs, |
| | ): |
| | super().__init__(**kwargs) |
| |
|
| | self.speech_encoder = speech_encoder |
| | self.speech_encoder_type = speech_encoder_type |
| | self.speech_projector_type = speech_projector_type |
| | self.speech_encoder_ds_rate = speech_encoder_ds_rate |
| | self.speech_encoder_hidden_size = speech_encoder_hidden_size |
| | self.mel_bins = mel_bins |
| | self.sample_rate = sample_rate |
| | self.frame_length = frame_length |
| | self.frame_shift = frame_shift |
| | self.use_beats = use_beats |
| | self.beats_model_path = beats_model_path |
| | self.whisper_config = whisper_config or {} |
| |
|
| | logger.info(f'Audio Config - Speech Encoder: {self.speech_encoder}') |
| | logger.info(f'Audio Config - Encoder Type: {self.speech_encoder_type}') |
| | logger.info(f'Audio Config - Projector Type: {self.speech_projector_type}') |
| | logger.info(f'Audio Config - Downsampling Rate: {self.speech_encoder_ds_rate}') |
| | logger.info(f'Audio Config - Hidden Size: {self.speech_encoder_hidden_size}') |
| | logger.info(f'Audio Config - Mel Bins: {self.mel_bins}') |
| | logger.info(f'Audio Config - Sample Rate: {self.sample_rate}') |
| |
|
| | @classmethod |
| | def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> 'PretrainedConfig': |
| | cls._set_token_in_kwargs(kwargs) |
| |
|
| | config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) |
| | return cls.from_dict(config_dict, **kwargs) |
| |
|
| | def to_dict(self): |
| | """ |
| | Serializes this instance to a Python dictionary. |
| | """ |
| | output = copy.deepcopy(self.__dict__) |
| | output['model_type'] = self.__class__.model_type |
| | return output |
| |
|
