from transformers import PretrainedConfig class AVHubertConfig(PretrainedConfig): model_type = "avhubert" def __init__( self, odim=5049, adim=1024, aheads=12, eunits=3072, elayers=12, transformer_input_layer="conv3d", dropout_rate=0.1, transformer_attn_dropout_rate=0.1, transformer_encoder_attn_layer_type="rel_mha", macaron_style=True, use_cnn_module=True, cnn_module_kernel=31, zero_triu=False, a_upsample_ratio=1, relu_type="swish", ddim=1024, dheads=16, dunits=3072, dlayers=6, lsm_weight=0.1, transformer_length_normalized_loss=False, mtlalpha=0.1, ctc_type="builtin", rel_pos_type="latest", fusion_hdim=8192, fusion_norm="batchnorm", hidden_size=1024, num_attention_heads=16, activation_dropout=0.0, activation_function="relu", adapter_attn_dim=None, adapter_kernel_size=3, adapter_stride=2, add_adapter=False, apply_spec_augment=True, attention_dropout=0.1, audio_dropout=0.5, audio_feat_dim=104, bos_token_id=1, classifier_proj_size=256, codevector_dim=256, contrastive_logits_temperature=0.1, conv_bias=False, conv_channels=1024, conv_dim=[512, 512, 512, 512, 512, 512, 512], conv_kernel=[10, 3, 3, 3, 3, 2, 2], conv_kernel_sizes=[5, 5], conv_stride=[5, 2, 2, 2, 2, 2, 2], ctc_loss_reduction="sum", ctc_zero_infinity=False, d_model=1024, decoder_attention_heads=8, decoder_ffn_dim=4096, decoder_layerdrop=0.0, decoder_layers=9, decoder_start_token_id=2, diversity_loss_weight=0.1, do_stable_layer_norm=False, dropout=0.1, dropout_features=0.1, dropout_input=0.1, encoder_attention_heads=16, encoder_embed_dim=1024, encoder_ffn_dim=2048, encoder_layerdrop=0.0, encoder_layers=12, eos_token_id=2, feat_extract_activation="gelu", feat_extract_norm="group", feat_proj_dropout=0.1, feat_quantizer_dropout=0.0, feature_grad_mult=0.1, final_dim=256, final_dropout=0.0, freeze_feat_extract_train=True, hidden_act="gelu", hidden_dropout=0.1, init_std=0.02, initializer_range=0.02, input_channels=1, input_feat_per_channel=80, intermediate_size=4096, is_encoder_decoder=True, label_rate=25, layer_norm_eps=1e-05, layerdrop=0.0, logit_temp=0.1, mask_channel_length=10, mask_channel_min_space=1, mask_channel_other=0.0, mask_channel_prob=0.0, mask_channel_selection="static", mask_feature_length=10, mask_feature_min_masks=0, mask_feature_prob=0.0, mask_length_audio=10, mask_length_image=5, mask_min_space=1, mask_other=0.0, mask_prob_audio=0.8, mask_prob_image=0.3, mask_selection="static", mask_time_length=10, mask_time_min_masks=2, mask_time_min_space=1, mask_time_other=0.0, mask_time_prob=0.0, mask_time_selection="static", masking_type="input", max_source_positions=6000, max_target_positions=2048, modality_dropout=0.5, modality_fuse="concat", modality="av", model_type="speech_to_text", no_mask_channel_overlap=False, no_mask_overlap=False, no_mask_time_overlap=False, num_adapter_layers=3, num_classes=2004, num_codevector_groups=2, num_codevectors_per_group=320, num_conv_layers=2, num_conv_pos_embedding_groups=16, num_conv_pos_embeddings=128, num_dictionaries=1, num_feat_extract_layers=7, num_hidden_layers=24, num_negatives=100, output_hidden_size=1024, pad_token_id=1, proj_codevector_dim=256, resnet_relu_type="prelu", resnet_weights=None, sample_rate=25, scale_embedding=None, selection_type="same_seq", sim_type="cosine", skip_masked=False, skip_nomask=False, sub_encoder_layers=0, target_glu=False, tdnn_dilation=[1, 2, 3, 1, 1], tdnn_dim=[512, 512, 512, 512, 1500], tdnn_kernel=[5, 3, 3, 1, 1], untie_final_proj=True, use_cache=True, use_weighted_layer_sum=False, vocab_size=1000, xvector_output_dim=512, **kwargs, ): super().__init__(**kwargs) self.odim = odim self.adim = adim self.aheads = aheads self.eunits = eunits self.elayers = elayers self.transformer_input_layer = transformer_input_layer self.dropout_rate = dropout_rate self.transformer_attn_dropout_rate = transformer_attn_dropout_rate self.transformer_encoder_attn_layer_type = transformer_encoder_attn_layer_type self.macaron_style = macaron_style self.use_cnn_module = use_cnn_module self.cnn_module_kernel = cnn_module_kernel self.zero_triu = zero_triu self.a_upsample_ratio = a_upsample_ratio self.relu_type = relu_type self.ddim = ddim self.dheads = dheads self.dunits = dunits self.dlayers = dlayers self.lsm_weight = lsm_weight self.transformer_length_normalized_loss = transformer_length_normalized_loss self.mtlalpha = mtlalpha self.ctc_type = ctc_type self.rel_pos_type = rel_pos_type self.fusion_hdim = fusion_hdim self.fusion_norm = fusion_norm self.hidden_size = hidden_size self.num_attention_heads = num_attention_heads self.activation_dropout = activation_dropout self.activation_function = activation_function self.adapter_attn_dim = adapter_attn_dim self.adapter_kernel_size = adapter_kernel_size self.adapter_stride = adapter_stride self.add_adapter = add_adapter self.apply_spec_augment = apply_spec_augment self.attention_dropout = attention_dropout self.audio_dropout = audio_dropout self.audio_feat_dim = audio_feat_dim self.bos_token_id = bos_token_id self.classifier_proj_size = classifier_proj_size self.codevector_dim = codevector_dim self.contrastive_logits_temperature = contrastive_logits_temperature self.conv_bias = conv_bias self.conv_channels = conv_channels self.conv_dim = conv_dim self.conv_kernel = conv_kernel self.conv_kernel_sizes = conv_kernel_sizes self.conv_stride = conv_stride self.ctc_loss_reduction = ctc_loss_reduction self.ctc_zero_infinity = ctc_zero_infinity self.d_model = d_model self.decoder_attention_heads = decoder_attention_heads self.decoder_ffn_dim = decoder_ffn_dim self.decoder_layerdrop = decoder_layerdrop self.decoder_layers = decoder_layers self.decoder_start_token_id = decoder_start_token_id self.diversity_loss_weight = diversity_loss_weight self.do_stable_layer_norm = do_stable_layer_norm self.dropout = dropout self.dropout_features = dropout_features self.dropout_input = dropout_input self.encoder_attention_heads = encoder_attention_heads self.encoder_embed_dim = encoder_embed_dim self.encoder_ffn_dim = encoder_ffn_dim self.encoder_layerdrop = encoder_layerdrop self.encoder_layers = encoder_layers self.eos_token_id = eos_token_id self.feat_extract_activation = feat_extract_activation self.feat_extract_norm = feat_extract_norm self.feat_proj_dropout = feat_proj_dropout self.feat_quantizer_dropout = feat_quantizer_dropout self.feature_grad_mult = feature_grad_mult self.final_dim = final_dim self.final_dropout = final_dropout self.freeze_feat_extract_train = freeze_feat_extract_train self.hidden_act = hidden_act self.hidden_dropout = hidden_dropout self.init_std = init_std self.initializer_range = initializer_range self.input_channels = input_channels self.input_feat_per_channel = input_feat_per_channel self.intermediate_size = intermediate_size self.is_encoder_decoder = is_encoder_decoder self.label_rate = label_rate self.layer_norm_eps = layer_norm_eps self.layerdrop = layerdrop self.logit_temp = logit_temp self.mask_channel_length = mask_channel_length self.mask_channel_min_space = mask_channel_min_space self.mask_channel_other = mask_channel_other self.mask_channel_prob = mask_channel_prob self.mask_channel_selection = mask_channel_selection self.mask_feature_length = mask_feature_length self.mask_feature_min_masks = mask_feature_min_masks self.mask_feature_prob = mask_feature_prob self.mask_length_audio = mask_length_audio self.mask_length_image = mask_length_image self.mask_min_space = mask_min_space self.mask_other = mask_other self.mask_prob_audio = mask_prob_audio self.mask_prob_image = mask_prob_image self.mask_selection = mask_selection self.mask_time_length = mask_time_length self.mask_time_min_masks = mask_time_min_masks self.mask_time_min_space = mask_time_min_space self.mask_time_other = mask_time_other self.mask_time_prob = mask_time_prob self.mask_time_selection = mask_time_selection self.masking_type = masking_type self.max_source_positions = max_source_positions self.max_target_positions = max_target_positions self.modality_dropout = modality_dropout self.modality_fuse = modality_fuse self.modality = modality self.model_type = model_type self.no_mask_channel_overlap = no_mask_channel_overlap self.no_mask_overlap = no_mask_overlap self.no_mask_time_overlap = no_mask_time_overlap self.num_adapter_layers = num_adapter_layers self.num_classes = num_classes self.num_codevector_groups = num_codevector_groups self.num_codevectors_per_group = num_codevectors_per_group self.num_conv_layers = num_conv_layers self.num_conv_pos_embedding_groups = num_conv_pos_embedding_groups self.num_conv_pos_embeddings = num_conv_pos_embeddings self.num_dictionaries = num_dictionaries self.num_feat_extract_layers = num_feat_extract_layers self.num_hidden_layers = num_hidden_layers self.num_negatives = num_negatives self.output_hidden_size = output_hidden_size self.pad_token_id = pad_token_id self.proj_codevector_dim = proj_codevector_dim self.resnet_relu_type = resnet_relu_type self.resnet_weights = resnet_weights self.sample_rate = sample_rate self.scale_embedding = scale_embedding self.selection_type = selection_type self.sim_type = sim_type self.skip_masked = skip_masked self.skip_nomask = skip_nomask self.sub_encoder_layers = sub_encoder_layers self.target_glu = target_glu self.tdnn_dilation = tdnn_dilation self.tdnn_dim = tdnn_dim self.tdnn_kernel = tdnn_kernel self.untie_final_proj = untie_final_proj self.use_cache = use_cache self.use_weighted_layer_sum = use_weighted_layer_sum self.vocab_size = vocab_size self.xvector_output_dim = xvector_output_dim