up

config.json

@@ -0,0 +1,79 @@
+{
+  "activation_dropout": 0.0,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForPreTraining"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "codevector_dim": 256,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": false,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": true,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_norm": "layer",
+  "feat_proj_dropout": 0.0,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.1,
+  "mask_feature_length": 10,
+  "mask_feature_prob": 0.1,
+  "mask_time_length": 10,
+  "mask_time_min_space": 1,
+  "mask_time_other": 0.0,
+  "mask_time_prob": 0.65,
+  "mask_time_selection": "static",
+  "model_type": "wav2vec2",
+  "no_mask_channel_overlap": false,
+  "no_mask_time_overlap": false,
+  "num_attention_heads": 12,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 12,
+  "num_negatives": 100,
+  "pad_token_id": 0,
+  "proj_codevector_dim": 256,
+  "transformers_version": "4.10.0.dev0",
+  "vocab_size": 32
+}

events.out.tfevents.1628809491.t1v-n-71556209-w-0.50858.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89c8f5cb3344090b39401a0808b6833df99ac31cc79ef21d479d80f9d8c6d548
+size 2322

events.out.tfevents.1628809788.t1v-n-71556209-w-0.52715.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55c10b47708f77daa1c0728421a9028decbea2da6cddf382a850083b1a4e389e
+size 6453

preprocessor_config.json

@@ -0,0 +1,8 @@
+{
+  "do_normalize": true,
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

run_main.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+./run_wav2vec2_pretrain_flax.py \
+    --output_dir="./" \
+    --num_train_epochs="2" \
+		--dataset_name="patrickvonplaten/librispeech_local" \
+		--dataset_config_name="clean" \
+		--train_split_name="train" \
+    --per_device_train_batch_size="16" \
+    --per_device_eval_batch_size="16" \
+    --learning_rate="5e-3" \
+    --weight_decay="0.01" \
+    --warmup_steps="1000" \
+    --model_name_or_path="./" \
+    --pad_to_multiple_of="16384" \
+		--logging_steps="2" \

run_wav2vec2_pretrain_flax.py

@@ -0,0 +1 @@
+/home/patrick/transformers/examples/research_projects/jax-projects/wav2vec2/run_wav2vec2_pretrain_flax.py