| from transformers import PretrainedConfig | |
| class AVHubertConfig(PretrainedConfig): | |
| model_type = "avhubert" | |
| def __init__( | |
| self, | |
| odim=5049, | |
| adim=1024, | |
| aheads=12, | |
| eunits=3072, | |
| elayers=12, | |
| transformer_input_layer="conv3d", | |
| dropout_rate=0.1, | |
| transformer_attn_dropout_rate=0.1, | |
| transformer_encoder_attn_layer_type="rel_mha", | |
| macaron_style=True, | |
| use_cnn_module=True, | |
| cnn_module_kernel=31, | |
| zero_triu=False, | |
| a_upsample_ratio=1, | |
| relu_type="swish", | |
| ddim=1024, | |
| dheads=16, | |
| dunits=3072, | |
| dlayers=6, | |
| lsm_weight=0.1, | |
| transformer_length_normalized_loss=False, | |
| mtlalpha=0.1, | |
| ctc_type="builtin", | |
| rel_pos_type="latest", | |
| fusion_hdim=8192, | |
| fusion_norm="batchnorm", | |
| hidden_size=1024, | |
| num_attention_heads=16, | |
| activation_dropout=0.0, | |
| activation_function="relu", | |
| adapter_attn_dim=None, | |
| adapter_kernel_size=3, | |
| adapter_stride=2, | |
| add_adapter=False, | |
| apply_spec_augment=True, | |
| attention_dropout=0.1, | |
| audio_dropout=0.5, | |
| audio_feat_dim=104, | |
| bos_token_id=1, | |
| classifier_proj_size=256, | |
| codevector_dim=256, | |
| contrastive_logits_temperature=0.1, | |
| conv_bias=False, | |
| conv_channels=1024, | |
| conv_dim=[512, 512, 512, 512, 512, 512, 512], | |
| conv_kernel=[10, 3, 3, 3, 3, 2, 2], | |
| conv_kernel_sizes=[5, 5], | |
| conv_stride=[5, 2, 2, 2, 2, 2, 2], | |
| ctc_loss_reduction="sum", | |
| ctc_zero_infinity=False, | |
| d_model=1024, | |
| decoder_attention_heads=8, | |
| decoder_ffn_dim=4096, | |
| decoder_layerdrop=0.0, | |
| decoder_layers=9, | |
| decoder_start_token_id=2, | |
| diversity_loss_weight=0.1, | |
| do_stable_layer_norm=False, | |
| dropout=0.1, | |
| dropout_features=0.1, | |
| dropout_input=0.1, | |
| encoder_attention_heads=16, | |
| encoder_embed_dim=1024, | |
| encoder_ffn_dim=2048, | |
| encoder_layerdrop=0.0, | |
| encoder_layers=12, | |
| eos_token_id=2, | |
| feat_extract_activation="gelu", | |
| feat_extract_norm="group", | |
| feat_proj_dropout=0.1, | |
| feat_quantizer_dropout=0.0, | |
| feature_grad_mult=0.1, | |
| final_dim=256, | |
| final_dropout=0.0, | |
| freeze_feat_extract_train=True, | |
| hidden_act="gelu", | |
| hidden_dropout=0.1, | |
| init_std=0.02, | |
| initializer_range=0.02, | |
| input_channels=1, | |
| input_feat_per_channel=80, | |
| intermediate_size=4096, | |
| is_encoder_decoder=True, | |
| label_rate=25, | |
| layer_norm_eps=1e-05, | |
| layerdrop=0.0, | |
| logit_temp=0.1, | |
| mask_channel_length=10, | |
| mask_channel_min_space=1, | |
| mask_channel_other=0.0, | |
| mask_channel_prob=0.0, | |
| mask_channel_selection="static", | |
| mask_feature_length=10, | |
| mask_feature_min_masks=0, | |
| mask_feature_prob=0.0, | |
| mask_length_audio=10, | |
| mask_length_image=5, | |
| mask_min_space=1, | |
| mask_other=0.0, | |
| mask_prob_audio=0.8, | |
| mask_prob_image=0.3, | |
| mask_selection="static", | |
| mask_time_length=10, | |
| mask_time_min_masks=2, | |
| mask_time_min_space=1, | |
| mask_time_other=0.0, | |
| mask_time_prob=0.0, | |
| mask_time_selection="static", | |
| masking_type="input", | |
| max_source_positions=6000, | |
| max_target_positions=2048, | |
| modality_dropout=0.5, | |
| modality_fuse="concat", | |
| modality="av", | |
| model_type="speech_to_text", | |
| no_mask_channel_overlap=False, | |
| no_mask_overlap=False, | |
| no_mask_time_overlap=False, | |
| num_adapter_layers=3, | |
| num_classes=2004, | |
| num_codevector_groups=2, | |
| num_codevectors_per_group=320, | |
| num_conv_layers=2, | |
| num_conv_pos_embedding_groups=16, | |
| num_conv_pos_embeddings=128, | |
| num_dictionaries=1, | |
| num_feat_extract_layers=7, | |
| num_hidden_layers=24, | |
| num_negatives=100, | |
| output_hidden_size=1024, | |
| pad_token_id=1, | |
| proj_codevector_dim=256, | |
| resnet_relu_type="prelu", | |
| resnet_weights=None, | |
| sample_rate=25, | |
| scale_embedding=None, | |
| selection_type="same_seq", | |
| sim_type="cosine", | |
| skip_masked=False, | |
| skip_nomask=False, | |
| sub_encoder_layers=0, | |
| target_glu=False, | |
| tdnn_dilation=[1, 2, 3, 1, 1], | |
| tdnn_dim=[512, 512, 512, 512, 1500], | |
| tdnn_kernel=[5, 3, 3, 1, 1], | |
| untie_final_proj=True, | |
| use_cache=True, | |
| use_weighted_layer_sum=False, | |
| vocab_size=1000, | |
| xvector_output_dim=512, | |
| **kwargs, | |
| ): | |
| super().__init__(**kwargs) | |
| self.odim = odim | |
| self.adim = adim | |
| self.aheads = aheads | |
| self.eunits = eunits | |
| self.elayers = elayers | |
| self.transformer_input_layer = transformer_input_layer | |
| self.dropout_rate = dropout_rate | |
| self.transformer_attn_dropout_rate = transformer_attn_dropout_rate | |
| self.transformer_encoder_attn_layer_type = transformer_encoder_attn_layer_type | |
| self.macaron_style = macaron_style | |
| self.use_cnn_module = use_cnn_module | |
| self.cnn_module_kernel = cnn_module_kernel | |
| self.zero_triu = zero_triu | |
| self.a_upsample_ratio = a_upsample_ratio | |
| self.relu_type = relu_type | |
| self.ddim = ddim | |
| self.dheads = dheads | |
| self.dunits = dunits | |
| self.dlayers = dlayers | |
| self.lsm_weight = lsm_weight | |
| self.transformer_length_normalized_loss = transformer_length_normalized_loss | |
| self.mtlalpha = mtlalpha | |
| self.ctc_type = ctc_type | |
| self.rel_pos_type = rel_pos_type | |
| self.fusion_hdim = fusion_hdim | |
| self.fusion_norm = fusion_norm | |
| self.hidden_size = hidden_size | |
| self.num_attention_heads = num_attention_heads | |
| self.activation_dropout = activation_dropout | |
| self.activation_function = activation_function | |
| self.adapter_attn_dim = adapter_attn_dim | |
| self.adapter_kernel_size = adapter_kernel_size | |
| self.adapter_stride = adapter_stride | |
| self.add_adapter = add_adapter | |
| self.apply_spec_augment = apply_spec_augment | |
| self.attention_dropout = attention_dropout | |
| self.audio_dropout = audio_dropout | |
| self.audio_feat_dim = audio_feat_dim | |
| self.bos_token_id = bos_token_id | |
| self.classifier_proj_size = classifier_proj_size | |
| self.codevector_dim = codevector_dim | |
| self.contrastive_logits_temperature = contrastive_logits_temperature | |
| self.conv_bias = conv_bias | |
| self.conv_channels = conv_channels | |
| self.conv_dim = conv_dim | |
| self.conv_kernel = conv_kernel | |
| self.conv_kernel_sizes = conv_kernel_sizes | |
| self.conv_stride = conv_stride | |
| self.ctc_loss_reduction = ctc_loss_reduction | |
| self.ctc_zero_infinity = ctc_zero_infinity | |
| self.d_model = d_model | |
| self.decoder_attention_heads = decoder_attention_heads | |
| self.decoder_ffn_dim = decoder_ffn_dim | |
| self.decoder_layerdrop = decoder_layerdrop | |
| self.decoder_layers = decoder_layers | |
| self.decoder_start_token_id = decoder_start_token_id | |
| self.diversity_loss_weight = diversity_loss_weight | |
| self.do_stable_layer_norm = do_stable_layer_norm | |
| self.dropout = dropout | |
| self.dropout_features = dropout_features | |
| self.dropout_input = dropout_input | |
| self.encoder_attention_heads = encoder_attention_heads | |
| self.encoder_embed_dim = encoder_embed_dim | |
| self.encoder_ffn_dim = encoder_ffn_dim | |
| self.encoder_layerdrop = encoder_layerdrop | |
| self.encoder_layers = encoder_layers | |
| self.eos_token_id = eos_token_id | |
| self.feat_extract_activation = feat_extract_activation | |
| self.feat_extract_norm = feat_extract_norm | |
| self.feat_proj_dropout = feat_proj_dropout | |
| self.feat_quantizer_dropout = feat_quantizer_dropout | |
| self.feature_grad_mult = feature_grad_mult | |
| self.final_dim = final_dim | |
| self.final_dropout = final_dropout | |
| self.freeze_feat_extract_train = freeze_feat_extract_train | |
| self.hidden_act = hidden_act | |
| self.hidden_dropout = hidden_dropout | |
| self.init_std = init_std | |
| self.initializer_range = initializer_range | |
| self.input_channels = input_channels | |
| self.input_feat_per_channel = input_feat_per_channel | |
| self.intermediate_size = intermediate_size | |
| self.is_encoder_decoder = is_encoder_decoder | |
| self.label_rate = label_rate | |
| self.layer_norm_eps = layer_norm_eps | |
| self.layerdrop = layerdrop | |
| self.logit_temp = logit_temp | |
| self.mask_channel_length = mask_channel_length | |
| self.mask_channel_min_space = mask_channel_min_space | |
| self.mask_channel_other = mask_channel_other | |
| self.mask_channel_prob = mask_channel_prob | |
| self.mask_channel_selection = mask_channel_selection | |
| self.mask_feature_length = mask_feature_length | |
| self.mask_feature_min_masks = mask_feature_min_masks | |
| self.mask_feature_prob = mask_feature_prob | |
| self.mask_length_audio = mask_length_audio | |
| self.mask_length_image = mask_length_image | |
| self.mask_min_space = mask_min_space | |
| self.mask_other = mask_other | |
| self.mask_prob_audio = mask_prob_audio | |
| self.mask_prob_image = mask_prob_image | |
| self.mask_selection = mask_selection | |
| self.mask_time_length = mask_time_length | |
| self.mask_time_min_masks = mask_time_min_masks | |
| self.mask_time_min_space = mask_time_min_space | |
| self.mask_time_other = mask_time_other | |
| self.mask_time_prob = mask_time_prob | |
| self.mask_time_selection = mask_time_selection | |
| self.masking_type = masking_type | |
| self.max_source_positions = max_source_positions | |
| self.max_target_positions = max_target_positions | |
| self.modality_dropout = modality_dropout | |
| self.modality_fuse = modality_fuse | |
| self.modality = modality | |
| self.model_type = model_type | |
| self.no_mask_channel_overlap = no_mask_channel_overlap | |
| self.no_mask_overlap = no_mask_overlap | |
| self.no_mask_time_overlap = no_mask_time_overlap | |
| self.num_adapter_layers = num_adapter_layers | |
| self.num_classes = num_classes | |
| self.num_codevector_groups = num_codevector_groups | |
| self.num_codevectors_per_group = num_codevectors_per_group | |
| self.num_conv_layers = num_conv_layers | |
| self.num_conv_pos_embedding_groups = num_conv_pos_embedding_groups | |
| self.num_conv_pos_embeddings = num_conv_pos_embeddings | |
| self.num_dictionaries = num_dictionaries | |
| self.num_feat_extract_layers = num_feat_extract_layers | |
| self.num_hidden_layers = num_hidden_layers | |
| self.num_negatives = num_negatives | |
| self.output_hidden_size = output_hidden_size | |
| self.pad_token_id = pad_token_id | |
| self.proj_codevector_dim = proj_codevector_dim | |
| self.resnet_relu_type = resnet_relu_type | |
| self.resnet_weights = resnet_weights | |
| self.sample_rate = sample_rate | |
| self.scale_embedding = scale_embedding | |
| self.selection_type = selection_type | |
| self.sim_type = sim_type | |
| self.skip_masked = skip_masked | |
| self.skip_nomask = skip_nomask | |
| self.sub_encoder_layers = sub_encoder_layers | |
| self.target_glu = target_glu | |
| self.tdnn_dilation = tdnn_dilation | |
| self.tdnn_dim = tdnn_dim | |
| self.tdnn_kernel = tdnn_kernel | |
| self.untie_final_proj = untie_final_proj | |
| self.use_cache = use_cache | |
| self.use_weighted_layer_sum = use_weighted_layer_sum | |
| self.vocab_size = vocab_size | |
| self.xvector_output_dim = xvector_output_dim | |