Training in progress, step 500

.gitignore

@@ -1,2 +1,3 @@
 checkpoint-*/
-.venv
+.venv
+1B/

README.md

@@ -1,61 +0,0 @@
----
-language:
-- et
-license: apache-2.0
-tags:
-- automatic-speech-recognition
-- mozilla-foundation/common_voice_7_0
-- generated_from_trainer
-datasets:
-- common_voice
-model-index:
-- name: ''
-  results: []
----
-
-<!-- This model card has been generated automatically according to the information the Trainer had access to. You
-should probably proofread and complete it, then remove this comment. -->
-
-# 
-
-This model is a fine-tuned version of [facebook/wav2vec2-xls-r-300m](https://huggingface.co/facebook/wav2vec2-xls-r-300m) on the MOZILLA-FOUNDATION/COMMON_VOICE_7_0 - ET dataset.
-It achieves the following results on the evaluation set:
-- Loss: 4.8300
-- Wer: 1.0
-
-## Model description
-
-More information needed
-
-## Intended uses & limitations
-
-More information needed
-
-## Training and evaluation data
-
-More information needed
-
-## Training procedure
-
-### Training hyperparameters
-
-The following hyperparameters were used during training:
-- learning_rate: 0.0003
-- train_batch_size: 8
-- eval_batch_size: 16
-- seed: 42
-- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
-- lr_scheduler_type: linear
-- training_steps: 10
-- mixed_precision_training: Native AMP
-
-### Training results
-
-
-
-### Framework versions
-
-- Transformers 4.16.0.dev0
-- Pytorch 1.10.1+cu102
-- Datasets 1.18.1.dev0
-- Tokenizers 0.11.0

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 37, "</s>": 38}

config.json

@@ -0,0 +1,107 @@
+{
+  "_name_or_path": "facebook/wav2vec2-xls-r-300m",
+  "activation_dropout": 0.0,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForCTC"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "codevector_dim": 768,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": true,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "mean",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": true,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_dropout": 0.0,
+  "feat_extract_norm": "layer",
+  "feat_proj_dropout": 0.1,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.0,
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.1,
+  "model_type": "wav2vec2",
+  "num_adapter_layers": 3,
+  "num_attention_heads": 16,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 24,
+  "num_negatives": 100,
+  "output_hidden_size": 1024,
+  "pad_token_id": 36,
+  "proj_codevector_dim": 768,
+  "tdnn_dilation": [
+    1,
+    2,
+    3,
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    512,
+    512,
+    512,
+    512,
+    1500
+  ],
+  "tdnn_kernel": [
+    5,
+    3,
+    3,
+    1,
+    1
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.16.0.dev0",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 39,
+  "xvector_output_dim": 512
+}

errs

@@ -0,0 +1,456 @@
+
+
+
+loading configuration file https://huggingface.co/facebook/wav2vec2-xls-r-300m/resolve/main/config.json from cache at /home/sagrilaft/.cache/huggingface/transformers/dabc27df63e37bd2a7a221c7774e35f36a280fbdf917cf54cadfc7df8c786f6f.a3e4c3c967d9985881e0ae550a5f6f668f897db5ab2e0802f9b97973b15970e6
+Model config Wav2Vec2Config {
+  "_name_or_path": "facebook/wav2vec2-xls-r-300m",
+  "activation_dropout": 0.0,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForPreTraining"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "codevector_dim": 768,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": true,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": true,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_dropout": 0.0,
+  "feat_extract_norm": "layer",
+  "feat_proj_dropout": 0.1,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.1,
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.075,
+  "model_type": "wav2vec2",
+  "num_adapter_layers": 3,
+  "num_attention_heads": 16,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 24,
+  "num_negatives": 100,
+  "output_hidden_size": 1024,
+  "pad_token_id": 0,
+  "proj_codevector_dim": 768,
+  "tdnn_dilation": [
+    1,
+    2,
+    3,
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    512,
+    512,
+    512,
+    512,
+    1500
+  ],
+  "tdnn_kernel": [
+    5,
+    3,
+    3,
+    1,
+    1
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.16.0.dev0",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 32,
+  "xvector_output_dim": 512
+}
+
+
  0%|          | 0/1 [00:00<?, ?ba/s]
+
  0%|          | 0/1 [00:00<?, ?ba/s]
+Didn't find file ./tokenizer_config.json. We won't load it.
+Didn't find file ./added_tokens.json. We won't load it.
+Didn't find file ./special_tokens_map.json. We won't load it.
+Didn't find file ./tokenizer.json. We won't load it.
+loading file ./vocab.json
+loading file None
+loading file None
+loading file None
+loading file None
+file ./config.json not found
+Adding <s> to the vocabulary
+Adding </s> to the vocabulary
+Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+loading configuration file https://huggingface.co/facebook/wav2vec2-xls-r-300m/resolve/main/config.json from cache at /home/sagrilaft/.cache/huggingface/transformers/dabc27df63e37bd2a7a221c7774e35f36a280fbdf917cf54cadfc7df8c786f6f.a3e4c3c967d9985881e0ae550a5f6f668f897db5ab2e0802f9b97973b15970e6
+Model config Wav2Vec2Config {
+  "_name_or_path": "facebook/wav2vec2-xls-r-300m",
+  "activation_dropout": 0.0,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForPreTraining"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "codevector_dim": 768,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": true,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": true,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_dropout": 0.0,
+  "feat_extract_norm": "layer",
+  "feat_proj_dropout": 0.1,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.1,
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.075,
+  "model_type": "wav2vec2",
+  "num_adapter_layers": 3,
+  "num_attention_heads": 16,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 24,
+  "num_negatives": 100,
+  "output_hidden_size": 1024,
+  "pad_token_id": 0,
+  "proj_codevector_dim": 768,
+  "tdnn_dilation": [
+    1,
+    2,
+    3,
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    512,
+    512,
+    512,
+    512,
+    1500
+  ],
+  "tdnn_kernel": [
+    5,
+    3,
+    3,
+    1,
+    1
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.16.0.dev0",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 32,
+  "xvector_output_dim": 512
+}
+
+loading feature extractor configuration file https://huggingface.co/facebook/wav2vec2-xls-r-300m/resolve/main/preprocessor_config.json from cache at /home/sagrilaft/.cache/huggingface/transformers/6fb028b95b394059e7d3b367bbca2382b576c66aebe896f04d2cd34e1b575f5b.d4484dc1c81456a2461485e7168b04347a7b9a4e3b1ef3aba723323b33e12326
+Feature extractor Wav2Vec2FeatureExtractor {
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}
+
+loading weights file https://huggingface.co/facebook/wav2vec2-xls-r-300m/resolve/main/pytorch_model.bin from cache at /home/sagrilaft/.cache/huggingface/transformers/1e6a6507f3b689035cd4b247e2a37c154e27f39143f31357a49b4e38baeccc36.1edb32803799e27ed554eb7dd935f6745b1a0b17b0ea256442fe24db6eb546cd
+Some weights of the model checkpoint at facebook/wav2vec2-xls-r-300m were not used when initializing Wav2Vec2ForCTC: ['project_q.bias', 'project_q.weight', 'quantizer.weight_proj.weight', 'project_hid.bias', 'project_hid.weight', 'quantizer.codevectors', 'quantizer.weight_proj.bias']
+- This IS expected if you are initializing Wav2Vec2ForCTC from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
+- This IS NOT expected if you are initializing Wav2Vec2ForCTC from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
+Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at facebook/wav2vec2-xls-r-300m and are newly initialized: ['lm_head.bias', 'lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+
+
  0%|          | 0/6 [00:00<?, ?ba/s]
+
  0%|          | 0/3 [00:00<?, ?ba/s]
+Configuration saved in ./preprocessor_config.json
+tokenizer config file saved in ./tokenizer_config.json
+Special tokens file saved in ./special_tokens_map.json
+added tokens file saved in ./added_tokens.json
+Configuration saved in ./config.json
+loading feature extractor configuration file ./preprocessor_config.json
+loading configuration file ./config.json
+Model config Wav2Vec2Config {
+  "_name_or_path": "./",
+  "activation_dropout": 0.0,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForPreTraining"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "codevector_dim": 768,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": true,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "mean",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": true,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_dropout": 0.0,
+  "feat_extract_norm": "layer",
+  "feat_proj_dropout": 0.1,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.0,
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.1,
+  "model_type": "wav2vec2",
+  "num_adapter_layers": 3,
+  "num_attention_heads": 16,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 24,
+  "num_negatives": 100,
+  "output_hidden_size": 1024,
+  "pad_token_id": 36,
+  "proj_codevector_dim": 768,
+  "tdnn_dilation": [
+    1,
+    2,
+    3,
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    512,
+    512,
+    512,
+    512,
+    1500
+  ],
+  "tdnn_kernel": [
+    5,
+    3,
+    3,
+    1,
+    1
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.16.0.dev0",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 39,
+  "xvector_output_dim": 512
+}
+
+loading feature extractor configuration file ./preprocessor_config.json
+Feature extractor Wav2Vec2FeatureExtractor {
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}
+
+Didn't find file ./tokenizer.json. We won't load it.
+loading file ./vocab.json
+loading file ./tokenizer_config.json
+loading file ./added_tokens.json
+loading file ./special_tokens_map.json
+loading file None
+Adding <s> to the vocabulary
+Adding </s> to the vocabulary
+/home/sagrilaft/Project/audio/xls-r-et/./ is already a clone of https://huggingface.co/shpotes/xls-r-et. Make sure you pull the latest changes with `repo.git_pull()`.
+Using amp half precision backend
+The following columns in the training set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running training *****
+  Num examples = 5705
+  Num Epochs = 500
+  Instantaneous batch size per device = 80
+  Total train batch size (w. parallel, distributed & accumulation) = 160
+  Gradient Accumulation steps = 2
+  Total optimization steps = 18000
+Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"
+wandb: Currently logged in as: shpotes (use `wandb login --relogin` to force relogin)
+wandb: Tracking run with wandb version 0.12.9
+wandb: Syncing run cosine+drop_proj+low_specaugment-300M
+wandb:  View project at https://wandb.ai/shpotes/xls-r-estonian
+wandb:  View run at https://wandb.ai/shpotes/xls-r-estonian/runs/1xdiy2kf
+wandb: Run data is saved locally in /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_105847-1xdiy2kf
+wandb: Run `wandb offline` to turn off syncing.
+
  0% 0/18000 [00:00<?, ?it/s]
  0% 1/18000 [00:17<86:26:02, 17.29s/it]
  0% 2/18000 [00:30<75:01:49, 15.01s/it]
  0% 3/18000 [00:41<65:24:50, 13.08s/it]
  0% 4/18000 [00:50<58:10:48, 11.64s/it]
  0% 5/18000 [00:59<52:10:20, 10.44s/it]
  0% 6/18000 [01:06<46:21:37,  9.28s/it]
  0% 7/18000 [01:12<40:36:25,  8.12s/it]
  0% 8/18000 [01:16<34:47:36,  6.96s/it]
  0% 9/18000 [01:33<50:28:39, 10.10s/it]
  0% 10/18000 [01:46<54:51:39, 10.98s/it]
  0% 11/18000 [01:57<54:52:13, 10.98s/it]
  0% 12/18000 [02:07<52:52:40, 10.58s/it]
  0% 13/18000 [02:15<49:41:21,  9.95s/it]
  0% 14/18000 [02:22<45:26:54,  9.10s/it]
  0% 15/18000 [02:28<40:51:40,  8.18s/it]
  0% 16/18000 [02:33<35:21:44,  7.08s/it]
  0% 17/18000 [02:49<49:42:15,  9.95s/it]
  0% 18/18000 [03:03<55:21:19, 11.08s/it]
  0% 19/18000 [03:14<55:04:47, 11.03s/it]
  0% 20/18000 [03:23<52:40:52, 10.55s/it]
  0% 21/18000 [03:32<49:32:32,  9.92s/it]
  0% 22/18000 [03:39<45:11:01,  9.05s/it]
  0% 23/18000 [03:45<40:31:29,  8.12s/it]
  0% 24/18000 [03:50<35:20:48,  7.08s/it]
  0% 25/18000 [04:06<49:22:23,  9.89s/it]
  0% 26/18000 [04:19<54:38:50, 10.95s/it]
  0% 27/18000 [04:30<54:41:00, 10.95s/it]
  0% 28/18000 [04:40<52:18:06, 10.48s/it]
  0% 29/18000 [04:48<49:00:52,  9.82s/it]
  0% 30/18000 [04:55<45:04:29,  9.03s/it]
  0% 31/18000 [05:01<40:27:51,  8.11s/it]
  0% 32/18000 [05:06<35:02:45,  7.02s/it]
  0% 33/18000 [05:20<46:37:19,  9.34s/it]
  0% 34/18000 [05:30<47:26:00,  9.50s/it]
  0% 35/18000 [05:37<43:56:15,  8.80s/it]
  0% 36/18000 [05:41<35:44:20,  7.16s/it]
  0% 37/18000 [05:57<49:56:45, 10.01s/it]
  0% 38/18000 [06:10<54:30:48, 10.93s/it]
  0% 39/18000 [06:21<54:14:33, 10.87s/it]
  0% 40/18000 [06:31<52:09:33, 10.46s/it]
  0% 41/18000 [06:39<49:11:45,  9.86s/it]
  0% 42/18000 [06:46<45:07:06,  9.04s/it]
  0% 43/18000 [06:52<40:22:51,  8.10s/it]
  0% 44/18000 [06:57<35:07:39,  7.04s/it]
  0% 45/18000 [07:13<48:38:37,  9.75s/it]
  0% 46/18000 [07:26<53:56:40, 10.82s/it]
  0% 47/18000 [07:37<53:51:56, 10.80s/it]
  0% 48/18000 [07:46<51:52:52, 10.40s/it]
  0% 49/18000 [07:55<48:44:50,  9.78s/it]
  0% 50/18000 [08:02<44:52:04,  9.00s/it]
  0% 51/18000 [08:08<40:35:01,  8.14s/it]
  0% 52/18000 [08:13<35:13:14,  7.06s/it]
  0% 53/18000 [08:29<49:34:56,  9.95s/it]
  0% 54/18000 [08:43<54:54:51, 11.02s/it]
  0% 55/18000 [08:54<54:39:48, 10.97s/it]
  0% 56/18000 [09:03<51:47:45, 10.39s/it]
  0% 57/18000 [09:11<48:30:24,  9.73s/it]
  0% 58/18000 [09:18<44:04:17,  8.84s/it]
  0% 59/18000 [09:23<39:07:38,  7.85s/it]
  0% 60/18000 [09:28<33:56:38,  6.81s/it]
  0% 61/18000 [09:44<48:31:20,  9.74s/it]
  0% 62/18000 [09:58<54:20:24, 10.91s/it]
  0% 63/18000 [10:09<54:07:29, 10.86s/it]
  0% 64/18000 [10:18<52:41:53, 10.58s/it]
  0% 65/18000 [10:27<49:44:17,  9.98s/it]
  0% 66/18000 [10:34<45:08:18,  9.06s/it]
  0% 67/18000 [10:40<40:27:18,  8.12s/it]
  0% 68/18000 [10:44<35:05:04,  7.04s/it]
  0% 69/18000 [11:00<47:40:09,  9.57s/it]
  0% 70/18000 [11:10<48:34:46,  9.75s/it]
  0% 71/18000 [11:17<44:54:05,  9.02s/it]
  0% 72/18000 [11:21<36:10:47,  7.27s/it]
  0% 73/18000 [11:37<50:11:00, 10.08s/it]
  0% 74/18000 [11:50<54:44:05, 10.99s/it]
  0% 75/18000 [12:01<54:03:33, 10.86s/it]
  0% 76/18000 [12:10<51:30:17, 10.34s/it]
  0% 77/18000 [12:18<48:16:12,  9.70s/it]
  0% 78/18000 [12:25<44:02:08,  8.85s/it]
  0% 79/18000 [12:31<39:24:36,  7.92s/it]
  0% 80/18000 [12:35<34:06:06,  6.85s/it]
  0% 81/18000 [12:52<48:23:03,  9.72s/it]
  0% 82/18000 [13:05<54:09:04, 10.88s/it]
  0% 83/18000 [13:16<54:31:03, 10.95s/it]
  0% 84/18000 [13:26<52:35:45, 10.57s/it]
  0% 85/18000 [13:34<49:34:08,  9.96s/it]
  0% 86/18000 [13:42<45:29:05,  9.14s/it]
  0% 87/18000 [13:48<40:44:29,  8.19s/it]
  0% 88/18000 [13:52<35:22:46,  7.11s/it]
  0% 89/18000 [14:09<49:08:37,  9.88s/it]
  0% 90/18000 [14:22<53:55:22, 10.84s/it]
  1% 91/18000 [14:33<53:54:12, 10.84s/it]
  1% 92/18000 [14:42<51:57:31, 10.45s/it]
  1% 93/18000 [14:50<48:11:46,  9.69s/it]
  1% 94/18000 [14:57<44:05:28,  8.86s/it]
  1% 95/18000 [15:02<39:02:47,  7.85s/it]
  1% 96/18000 [15:07<33:46:57,  6.79s/it]
  1% 97/18000 [15:23<48:11:08,  9.69s/it]
  1% 98/18000 [15:36<53:29:06, 10.76s/it]
  1% 99/18000 [15:47<53:25:40, 10.74s/it]
  1% 100/18000 [15:57<51:48:37, 10.42s/it]
                                          
  1% 100/18000 [15:57<51:48:37, 10.42s/it]
  1% 101/18000 [16:05<48:40:04,  9.79s/it]
  1% 102/18000 [16:12<44:23:01,  8.93s/it]
  1% 103/18000 [16:18<39:26:24,  7.93s/it]
  1% 104/18000 [16:22<34:15:20,  6.89s/it]
  1% 105/18000 [16:37<46:43:13,  9.40s/it]
  1% 106/18000 [16:47<47:23:12,  9.53s/it]
  1% 107/18000 [16:54<43:44:40,  8.80s/it]
  1% 108/18000 [16:58<35:46:46,  7.20s/it]
  1% 109/18000 [17:14<48:43:15,  9.80s/it]
  1% 110/18000 [17:27<54:09:47, 10.90s/it]
  1% 111/18000 [17:38<54:15:19, 10.92s/it]
  1% 112/18000 [17:48<52:17:25, 10.52s/it]
  1% 113/18000 [17:56<49:14:31,  9.91s/it]
  1% 114/18000 [18:03<45:07:53,  9.08s/it]
  1% 115/18000 [18:09<40:13:03,  8.10s/it]
  1% 116/18000 [18:13<34:43:29,  6.99s/it]
  1% 117/18000 [18:30<48:46:18,  9.82s/it]
  1% 118/18000 [18:43<53:31:11, 10.77s/it]
  1% 119/18000 [18:53<53:11:36, 10.71s/it]
  1% 120/18000 [19:03<50:56:21, 10.26s/it]
  1% 121/18000 [19:11<47:23:37,  9.54s/it]
  1% 122/18000 [19:17<43:07:59,  8.69s/it]
  1% 123/18000 [19:23<38:34:05,  7.77s/it]
  1% 124/18000 [19:27<33:23:03,  6.72s/it]
  1% 125/18000 [19:43<47:43:04,  9.61s/it]
  1% 126/18000 [19:57<53:09:48, 10.71s/it]
  1% 127/18000 [20:08<53:19:09, 10.74s/it]
  1% 128/18000 [20:17<51:20:03, 10.34s/it]
  1% 129/18000 [20:25<48:12:21,  9.71s/it]
  1% 130/18000 [20:32<43:58:10,  8.86s/it]
  1% 131/18000 [20:38<39:28:05,  7.95s/it]
  1% 132/18000 [20:42<34:15:06,  6.90s/it]
  1% 133/18000 [20:59<48:28:40,  9.77s/it]
  1% 134/18000 [21:12<53:33:44, 10.79s/it]
  1% 135/18000 [21:23<53:23:03, 10.76s/it]
  1% 136/18000 [21:32<51:19:46, 10.34s/it]
  1% 137/18000 [21:40<48:04:44,  9.69s/it]
  1% 138/18000 [21:47<43:54:30,  8.85s/it]
  1% 139/18000 [21:53<39:05:17,  7.88s/it]
  1% 140/18000 [21:57<34:00:50,  6.86s/it]
  1% 141/18000 [22:13<46:40:32,  9.41s/it]
  1% 142/18000 [22:23<47:51:15,  9.65s/it]
  1% 143/18000 [22:30<44:22:27,  8.95s/it]
  1% 144/18000 [22:34<36:18:44,  7.32s/it]
  1% 145/18000 [22:50<49:56:40, 10.07s/it]
  1% 146/18000 [23:03<54:50:33, 11.06s/it]
  1% 147/18000 [23:14<54:23:30, 10.97s/it]
  1% 148/18000 [23:24<51:53:58, 10.47s/it]
  1% 149/18000 [23:32<48:32:27,  9.79s/it]
  1% 150/18000 [23:39<44:37:06,  9.00s/it]
  1% 151/18000 [23:45<39:56:07,  8.05s/it]
  1% 152/18000 [23:49<34:30:40,  6.96s/it]
  1% 153/18000 [24:06<48:51:50,  9.86s/it]
  1% 154/18000 [24:19<54:08:29, 10.92s/it]
  1% 155/18000 [24:30<54:07:19, 10.92s/it]
  1% 156/18000 [24:39<51:49:42, 10.46s/it]
  1% 157/18000 [24:48<48:49:19,  9.85s/it]
  1% 158/18000 [24:55<44:54:15,  9.06s/it]
  1% 159/18000 [25:01<40:01:43,  8.08s/it]
  1% 160/18000 [25:05<34:36:26,  6.98s/it]
  1% 161/18000 [25:22<48:39:48,  9.82s/it]
  1% 162/18000 [25:35<53:20:53, 10.77s/it]
  1% 163/18000 [25:45<53:02:40, 10.71s/it]
  1% 164/18000 [25:55<51:11:53, 10.33s/it]
  1% 165/18000 [26:03<48:01:45,  9.69s/it]
  1% 166/18000 [26:10<43:51:38,  8.85s/it]
  1% 167/18000 [26:16<39:20:09,  7.94s/it]
  1% 168/18000 [26:20<34:07:20,  6.89s/it]
  1% 169/18000 [26:37<48:26:45,  9.78s/it]
  1% 170/18000 [26:50<54:05:34, 10.92s/it]
  1% 171/18000 [27:01<54:10:19, 10.94s/it]
  1% 172/18000 [27:11<51:47:16, 10.46s/it]
  1% 173/18000 [27:19<48:35:45,  9.81s/it]
  1% 174/18000 [27:26<44:17:21,  8.94s/it]
  1% 175/18000 [27:32<39:40:01,  8.01s/it]
  1% 176/18000 [27:36<34:26:47,  6.96s/it]
  1% 177/18000 [27:51<46:22:02,  9.37s/it]
  1% 178/18000 [28:01<46:54:42,  9.48s/it]
  1% 179/18000 [28:08<43:12:54,  8.73s/it]
  1% 180/18000 [28:11<34:51:46,  7.04s/it]
  1% 181/18000 [28:27<48:41:49,  9.84s/it]
  1% 182/18000 [28:41<53:47:06, 10.87s/it]
  1% 183/18000 [28:51<53:26:11, 10.80s/it]
  1% 184/18000 [29:00<51:13:34, 10.35s/it]
  1% 185/18000 [29:08<47:40:37,  9.63s/it]
  1% 186/18000 [29:15<43:41:55,  8.83s/it]
  1% 187/18000 [29:21<39:02:36,  7.89s/it]
  1% 188/18000 [29:25<33:33:56,  6.78s/it]
  1% 189/18000 [29:41<46:58:38,  9.50s/it]
  1% 190/18000 [29:55<52:50:48, 10.68s/it]
  1% 191/18000 [30:05<53:13:50, 10.76s/it]
  1% 192/18000 [30:15<51:19:04, 10.37s/it]
  1% 193/18000 [30:23<47:46:28,  9.66s/it]
  1% 194/18000 [30:30<43:50:18,  8.86s/it]
  1% 195/18000 [30:36<39:23:19,  7.96s/it]
  1% 196/18000 [30:40<34:17:27,  6.93s/it]
  1% 197/18000 [30:57<48:42:25,  9.85s/it]
  1% 198/18000 [31:11<54:34:28, 11.04s/it]
  1% 199/18000 [31:22<54:40:44, 11.06s/it]
  1% 200/18000 [31:32<53:10:28, 10.75s/it]
                                          
  1% 200/18000 [31:32<53:10:28, 10.75s/it]
  1% 201/18000 [31:40<49:47:14, 10.07s/it]
  1% 202/18000 [31:47<45:18:07,  9.16s/it]
  1% 203/18000 [31:53<40:20:20,  8.16s/it]
  1% 204/18000 [31:58<34:46:55,  7.04s/it]
  1% 205/18000 [32:14<48:45:25,  9.86s/it]
  1% 206/18000 [32:27<53:32:26, 10.83s/it]
  1% 207/18000 [32:38<53:17:02, 10.78s/it]
  1% 208/18000 [32:47<51:12:19, 10.36s/it]
  1% 209/18000 [32:56<48:09:33,  9.75s/it]
  1% 210/18000 [33:03<44:08:23,  8.93s/it]
  1% 211/18000 [33:09<39:35:55,  8.01s/it]
  1% 212/18000 [33:13<34:23:10,  6.96s/it]
  1% 213/18000 [33:28<46:28:43,  9.41s/it]
  1% 214/18000 [33:38<46:56:12,  9.50s/it]
  1% 215/18000 [33:45<43:27:58,  8.80s/it]
  1% 216/18000 [33:48<35:04:07,  7.10s/it]
  1% 217/18000 [34:04<48:42:58,  9.86s/it]
  1% 218/18000 [34:18<53:32:18, 10.84s/it]
  1% 219/18000 [34:28<53:14:19, 10.78s/it]
  1% 220/18000 [34:38<51:06:07, 10.35s/it]
  1% 221/18000 [34:46<47:42:29,  9.66s/it]
  1% 222/18000 [34:52<43:16:38,  8.76s/it]
  1% 223/18000 [34:58<38:47:18,  7.85s/it]
  1% 224/18000 [35:02<33:35:15,  6.80s/it]
  1% 225/18000 [35:18<46:55:15,  9.50s/it]
  1% 226/18000 [35:32<52:36:24, 10.66s/it]
  1% 227/18000 [35:42<52:47:45, 10.69s/it]
  1% 228/18000 [35:52<50:52:21, 10.31s/it]
  1% 229/18000 [36:00<48:08:38,  9.75s/it]
  1% 230/18000 [36:07<44:14:44,  8.96s/it]
  1% 231/18000 [36:13<39:37:28,  8.03s/it]
  1% 232/18000 [36:18<34:13:31,  6.93s/it]
  1% 233/18000 [36:34<48:28:27,  9.82s/it]
  1% 234/18000 [36:48<53:55:28, 10.93s/it]
  1% 235/18000 [36:59<54:07:48, 10.97s/it]
  1% 236/18000 [37:08<52:09:20, 10.57s/it]
  1% 237/18000 [37:17<48:42:27,  9.87s/it]
  1% 238/18000 [37:23<44:21:14,  8.99s/it]
  1% 239/18000 [37:29<39:44:51,  8.06s/it]
  1% 240/18000 [37:34<34:23:30,  6.97s/it]
  1% 241/18000 [37:50<48:42:06,  9.87s/it]
  1% 242/18000 [38:03<53:25:27, 10.83s/it]
  1% 243/18000 [38:14<52:55:48, 10.73s/it]
  1% 244/18000 [38:23<50:37:01, 10.26s/it]
  1% 245/18000 [38:31<47:27:12,  9.62s/it]
  1% 246/18000 [38:38<43:25:26,  8.81s/it]
  1% 247/18000 [38:44<39:01:44,  7.91s/it]
  1% 248/18000 [38:48<33:54:20,  6.88s/it]
  1% 249/18000 [39:04<46:20:55,  9.40s/it]
  1% 250/18000 [39:14<47:17:59,  9.59s/it]
  1% 251/18000 [39:21<43:48:05,  8.88s/it]
  1% 252/18000 [39:24<35:17:51,  7.16s/it]
  1% 253/18000 [39:41<49:42:54, 10.08s/it]
  1% 254/18000 [39:55<54:40:20, 11.09s/it]
  1% 255/18000 [40:05<54:05:56, 10.98s/it]
  1% 256/18000 [40:15<51:41:45, 10.49s/it]
  1% 257/18000 [40:23<48:34:31,  9.86s/it]
  1% 258/18000 [40:30<44:23:42,  9.01s/it]
  1% 259/18000 [40:36<39:40:19,  8.05s/it]
  1% 260/18000 [40:40<34:13:42,  6.95s/it]
  1% 261/18000 [40:57<48:18:11,  9.80s/it]
  1% 262/18000 [41:10<53:34:01, 10.87s/it]
  1% 263/18000 [41:21<53:19:57, 10.82s/it]
  1% 264/18000 [41:30<50:44:07, 10.30s/it]
  1% 265/18000 [41:38<47:15:09,  9.59s/it]
  1% 266/18000 [41:44<43:02:02,  8.74s/it]
  1% 267/18000 [41:50<38:41:55,  7.86s/it]
  1% 268/18000 [41:55<33:41:12,  6.84s/it]
  1% 269/18000 [42:11<47:53:19,  9.72s/it]
  2% 270/18000 [42:24<53:01:13, 10.77s/it]
  2% 271/18000 [42:35<53:02:40, 10.77s/it]
  2% 272/18000 [42:45<51:00:00, 10.36s/it]
  2% 273/18000 [42:53<48:01:13,  9.75s/it]
  2% 274/18000 [43:00<44:05:26,  8.95s/it]
  2% 275/18000 [43:06<39:29:31,  8.02s/it]
  2% 276/18000 [43:10<34:20:22,  6.97s/it]
  2% 277/18000 [43:27<48:36:20,  9.87s/it]
  2% 278/18000 [43:41<54:13:56, 11.02s/it]
  2% 279/18000 [43:52<54:12:18, 11.01s/it]
  2% 280/18000 [44:01<51:45:06, 10.51s/it]
  2% 281/18000 [44:10<48:47:06,  9.91s/it]
  2% 282/18000 [44:16<44:16:39,  9.00s/it]
  2% 283/18000 [44:22<39:08:42,  7.95s/it]
  2% 284/18000 [44:26<33:59:45,  6.91s/it]
  2% 285/18000 [44:42<46:30:46,  9.45s/it]
  2% 286/18000 [44:52<47:30:24,  9.65s/it]
  2% 287/18000 [44:59<43:54:19,  8.92s/it]
  2% 288/18000 [45:03<35:52:00,  7.29s/it]
  2% 289/18000 [45:19<49:57:55, 10.16s/it]
  2% 290/18000 [45:33<54:54:58, 11.16s/it]
  2% 291/18000 [45:44<54:40:30, 11.11s/it]
  2% 292/18000 [45:54<52:24:02, 10.65s/it]
  2% 293/18000 [46:02<48:53:00,  9.94s/it]
  2% 294/18000 [46:09<44:31:18,  9.05s/it]
  2% 295/18000 [46:15<39:58:45,  8.13s/it]
  2% 296/18000 [46:19<34:41:14,  7.05s/it]
  2% 297/18000 [46:36<48:25:58,  9.85s/it]
  2% 298/18000 [46:49<53:21:33, 10.85s/it]
  2% 299/18000 [47:00<53:00:52, 10.78s/it]
  2% 300/18000 [47:09<51:02:08, 10.38s/it]
                                          
  2% 300/18000 [47:09<51:02:08, 10.38s/it]
  2% 301/18000 [47:17<47:51:13,  9.73s/it]
  2% 302/18000 [47:24<43:43:29,  8.89s/it]
  2% 303/18000 [47:30<39:03:22,  7.94s/it]
  2% 304/18000 [47:34<33:47:15,  6.87s/it]
  2% 305/18000 [47:51<47:55:18,  9.75s/it]
  2% 306/18000 [48:04<52:59:15, 10.78s/it]
  2% 307/18000 [48:15<52:51:07, 10.75s/it]
  2% 308/18000 [48:24<50:25:56, 10.26s/it]
  2% 309/18000 [48:32<47:33:47,  9.68s/it]
  2% 310/18000 [48:39<43:29:15,  8.85s/it]
  2% 311/18000 [48:45<38:43:45,  7.88s/it]
  2% 312/18000 [48:49<33:28:13,  6.81s/it]
  2% 313/18000 [49:05<47:37:14,  9.69s/it]
  2% 314/18000 [49:19<53:06:18, 10.81s/it]
  2% 315/18000 [49:29<52:32:02, 10.69s/it]
  2% 316/18000 [49:39<50:37:01, 10.30s/it]
  2% 317/18000 [49:47<47:35:40,  9.69s/it]
  2% 318/18000 [49:54<43:33:20,  8.87s/it]
  2% 319/18000 [50:00<39:14:50,  7.99s/it]
  2% 320/18000 [50:04<34:09:36,  6.96s/it]
  2% 321/18000 [50:19<46:07:44,  9.39s/it]
  2% 322/18000 [50:29<47:04:05,  9.59s/it]
  2% 323/18000 [50:37<43:38:37,  8.89s/it]
  2% 324/18000 [50:40<35:28:44,  7.23s/it]
  2% 325/18000 [50:56<49:06:27, 10.00s/it]
  2% 326/18000 [51:10<53:47:16, 10.96s/it]
  2% 327/18000 [51:20<53:23:03, 10.87s/it]
  2% 328/18000 [51:30<51:14:24, 10.44s/it]
  2% 329/18000 [51:38<48:08:52,  9.81s/it]
  2% 330/18000 [51:45<44:04:02,  8.98s/it]
  2% 331/18000 [51:51<39:27:52,  8.04s/it]
  2% 332/18000 [51:55<34:01:15,  6.93s/it]
  2% 333/18000 [52:12<47:55:08,  9.76s/it]
  2% 334/18000 [52:25<52:58:37, 10.80s/it]
  2% 335/18000 [52:36<53:03:41, 10.81s/it]
  2% 336/18000 [52:45<51:22:12, 10.47s/it]
  2% 337/18000 [52:54<48:17:41,  9.84s/it]
  2% 338/18000 [53:01<44:04:01,  8.98s/it]
  2% 339/18000 [53:06<39:19:19,  8.02s/it]
  2% 340/18000 [53:11<33:59:49,  6.93s/it]
  2% 341/18000 [53:28<48:17:19,  9.84s/it]
  2% 342/18000 [53:40<52:48:55, 10.77s/it]
  2% 343/18000 [53:51<52:45:57, 10.76s/it]
  2% 344/18000 [54:00<50:32:06, 10.30s/it]
  2% 345/18000 [54:09<47:20:31,  9.65s/it]
  2% 346/18000 [54:15<43:17:34,  8.83s/it]
  2% 347/18000 [54:21<38:41:00,  7.89s/it]
  2% 348/18000 [54:25<33:24:17,  6.81s/it]
  2% 349/18000 [54:42<47:36:12,  9.71s/it]
  2% 350/18000 [54:55<52:56:31, 10.80s/it]
  2% 351/18000 [55:05<52:05:17, 10.62s/it]
  2% 352/18000 [55:15<49:57:37, 10.19s/it]
  2% 353/18000 [55:23<46:53:40,  9.57s/it]
  2% 354/18000 [55:30<42:56:28,  8.76s/it]
  2% 355/18000 [55:35<38:35:32,  7.87s/it]
  2% 356/18000 [55:40<33:34:24,  6.85s/it]
  2% 357/18000 [55:55<46:02:43,  9.40s/it]
  2% 358/18000 [56:05<47:14:04,  9.64s/it]
  2% 359/18000 [56:13<43:54:21,  8.96s/it]
  2% 360/18000 [56:16<35:44:58,  7.30s/it]
  2% 361/18000 [56:33<49:37:45, 10.13s/it]
  2% 362/18000 [56:46<54:18:29, 11.08s/it]
  2% 363/18000 [56:57<53:18:49, 10.88s/it]
  2% 364/18000 [57:06<50:43:27, 10.35s/it]
  2% 365/18000 [57:14<47:50:00,  9.76s/it]
  2% 366/18000 [57:21<43:42:30,  8.92s/it]
  2% 367/18000 [57:27<38:47:25,  7.92s/it]
  2% 368/18000 [57:31<33:26:40,  6.83s/it]
  2% 369/18000 [57:47<47:20:50,  9.67s/it]
  2% 370/18000 [58:01<52:33:21, 10.73s/it]
  2% 371/18000 [58:11<52:40:27, 10.76s/it]
  2% 372/18000 [58:21<50:37:46, 10.34s/it]
  2% 373/18000 [58:29<47:13:33,  9.65s/it]
  2% 374/18000 [58:36<43:15:43,  8.84s/it]
  2% 375/18000 [58:41<38:46:26,  7.92s/it]
  2% 376/18000 [58:46<33:49:26,  6.91s/it]
  2% 377/18000 [59:03<47:58:02,  9.80s/it]
  2% 378/18000 [59:16<53:14:23, 10.88s/it]
  2% 379/18000 [59:27<53:21:26, 10.90s/it]
  2% 380/18000 [59:37<51:32:24, 10.53s/it]
  2% 381/18000 [59:45<48:23:16,  9.89s/it]
  2% 382/18000 [59:52<44:25:48,  9.08s/it]
  2% 383/18000 [59:58<39:54:39,  8.16s/it]
  2% 384/18000 [1:00:03<34:22:55,  7.03s/it]
  2% 385/18000 [1:00:19<48:14:08,  9.86s/it]
  2% 386/18000 [1:00:32<53:13:30, 10.88s/it]
  2% 387/18000 [1:00:43<52:53:09, 10.81s/it]
  2% 388/18000 [1:00:52<50:27:17, 10.31s/it]
  2% 389/18000 [1:01:00<47:11:15,  9.65s/it]
  2% 390/18000 [1:01:07<43:01:58,  8.80s/it]
  2% 391/18000 [1:01:13<38:27:51,  7.86s/it]
  2% 392/18000 [1:01:17<33:21:31,  6.82s/it]
  2% 393/18000 [1:01:32<44:31:41,  9.10s/it]
  2% 394/18000 [1:01:41<45:48:47,  9.37s/it]
  2% 395/18000 [1:01:49<42:48:03,  8.75s/it]
  2% 396/18000 [1:01:52<34:58:38,  7.15s/it]
  2% 397/18000 [1:02:09<48:50:53,  9.99s/it]
  2% 398/18000 [1:02:22<53:43:53, 10.99s/it]
  2% 399/18000 [1:02:33<53:20:56, 10.91s/it]
  2% 400/18000 [1:02:42<51:16:29, 10.49s/it]
                                            
  2% 400/18000 [1:02:42<51:16:29, 10.49s/it]
  2% 401/18000 [1:02:51<48:04:04,  9.83s/it]
  2% 402/18000 [1:02:58<43:47:40,  8.96s/it]
  2% 403/18000 [1:03:03<39:11:36,  8.02s/it]
  2% 404/18000 [1:03:08<33:59:02,  6.95s/it]
  2% 405/18000 [1:03:24<47:36:15,  9.74s/it]
  2% 406/18000 [1:03:37<52:47:43, 10.80s/it]
  2% 407/18000 [1:03:48<52:48:26, 10.81s/it]
  2% 408/18000 [1:03:58<50:44:50, 10.38s/it]
  2% 409/18000 [1:04:06<47:49:35,  9.79s/it]
  2% 410/18000 [1:04:13<43:57:02,  9.00s/it]
  2% 411/18000 [1:04:19<39:21:08,  8.05s/it]
  2% 412/18000 [1:04:23<33:57:31,  6.95s/it]
  2% 413/18000 [1:04:40<47:43:17,  9.77s/it]
  2% 414/18000 [1:04:53<53:05:33, 10.87s/it]
  2% 415/18000 [1:05:04<53:03:39, 10.86s/it]
  2% 416/18000 [1:05:14<51:10:13, 10.48s/it]
  2% 417/18000 [1:05:22<47:58:19,  9.82s/it]
  2% 418/18000 [1:05:29<43:46:49,  8.96s/it]
  2% 419/18000 [1:05:35<39:10:03,  8.02s/it]
  2% 420/18000 [1:05:39<33:58:45,  6.96s/it]
  2% 421/18000 [1:05:56<48:16:48,  9.89s/it]
  2% 422/18000 [1:06:09<53:30:55, 10.96s/it]
  2% 423/18000 [1:06:20<53:13:22, 10.90s/it]
  2% 424/18000 [1:06:29<50:26:10, 10.33s/it]
  2% 425/18000 [1:06:37<47:08:56,  9.66s/it]
  2% 426/18000 [1:06:44<43:19:15,  8.87s/it]
  2% 427/18000 [1:06:50<38:33:18,  7.90s/it]
  2% 428/18000 [1:06:54<33:33:27,  6.88s/it]
  2% 429/18000 [1:07:09<44:43:15,  9.16s/it]
  2% 430/18000 [1:07:19<46:00:07,  9.43s/it]
  2% 431/18000 [1:07:26<43:11:33,  8.85s/it]
  2% 432/18000 [1:07:30<34:56:30,  7.16s/it]
  2% 433/18000 [1:07:46<48:49:24, 10.01s/it]
  2% 434/18000 [1:08:00<53:38:05, 10.99s/it]
  2% 435/18000 [1:08:10<53:09:03, 10.89s/it]
  2% 436/18000 [1:08:19<50:38:26, 10.38s/it]
  2% 437/18000 [1:08:27<47:14:39,  9.68s/it]
  2% 438/18000 [1:08:34<42:52:00,  8.79s/it]
  2% 439/18000 [1:08:40<38:20:18,  7.86s/it]
  2% 440/18000 [1:08:44<33:10:31,  6.80s/it]
  2% 441/18000 [1:09:01<47:29:55,  9.74s/it]
  2% 442/18000 [1:09:14<52:47:34, 10.82s/it]
  2% 443/18000 [1:09:25<52:43:24, 10.81s/it]
  2% 444/18000 [1:09:34<50:46:19, 10.41s/it]
  2% 445/18000 [1:09:43<47:42:44,  9.78s/it]
  2% 446/18000 [1:09:50<43:40:10,  8.96s/it]
  2% 447/18000 [1:09:56<39:20:53,  8.07s/it]
  2% 448/18000 [1:10:00<34:14:21,  7.02s/it]
  2% 449/18000 [1:10:17<48:02:46,  9.86s/it]
  2% 450/18000 [1:10:30<53:38:52, 11.00s/it]
  3% 451/18000 [1:10:41<53:33:23, 10.99s/it]
  3% 452/18000 [1:10:51<51:28:10, 10.56s/it]
  3% 453/18000 [1:10:59<47:59:20,  9.85s/it]
  3% 454/18000 [1:11:06<43:39:42,  8.96s/it]
  3% 455/18000 [1:11:12<39:03:03,  8.01s/it]
  3% 456/18000 [1:11:16<33:48:03,  6.94s/it]
  3% 457/18000 [1:11:33<47:28:06,  9.74s/it]
  3% 458/18000 [1:11:46<52:13:26, 10.72s/it]
  3% 459/18000 [1:11:56<52:11:58, 10.71s/it]
  3% 460/18000 [1:12:06<50:26:50, 10.35s/it]
  3% 461/18000 [1:12:14<47:11:11,  9.69s/it]
  3% 462/18000 [1:12:21<43:22:01,  8.90s/it]
  3% 463/18000 [1:12:27<38:48:08,  7.97s/it]
  3% 464/18000 [1:12:31<33:40:40,  6.91s/it]
  3% 465/18000 [1:12:46<45:41:28,  9.38s/it]
  3% 466/18000 [1:12:56<46:32:41,  9.56s/it]
  3% 467/18000 [1:13:03<43:03:28,  8.84s/it]
  3% 468/18000 [1:13:07<34:43:53,  7.13s/it]
  3% 469/18000 [1:13:23<48:48:43, 10.02s/it]
  3% 470/18000 [1:13:37<54:05:41, 11.11s/it]
  3% 471/18000 [1:13:48<53:57:52, 11.08s/it]
  3% 472/18000 [1:13:58<51:57:14, 10.67s/it]
  3% 473/18000 [1:14:06<48:17:17,  9.92s/it]
  3% 474/18000 [1:14:13<44:14:22,  9.09s/it]
  3% 475/18000 [1:14:19<39:36:15,  8.14s/it]
  3% 476/18000 [1:14:24<34:21:02,  7.06s/it]
  3% 477/18000 [1:14:40<47:52:39,  9.84s/it]
  3% 478/18000 [1:14:53<52:38:58, 10.82s/it]
  3% 479/18000 [1:15:04<52:32:09, 10.79s/it]
  3% 480/18000 [1:15:13<50:33:14, 10.39s/it]
  3% 481/18000 [1:15:22<47:38:56,  9.79s/it]
  3% 482/18000 [1:15:28<43:19:51,  8.90s/it]
  3% 483/18000 [1:15:34<38:46:14,  7.97s/it]
  3% 484/18000 [1:15:39<33:35:21,  6.90s/it]
  3% 485/18000 [1:15:55<47:26:03,  9.75s/it]
  3% 486/18000 [1:16:08<52:32:49, 10.80s/it]
  3% 487/18000 [1:16:19<52:38:09, 10.82s/it]
  3% 488/18000 [1:16:28<50:31:07, 10.39s/it]
  3% 489/18000 [1:16:37<47:24:15,  9.75s/it]
  3% 490/18000 [1:16:44<43:32:47,  8.95s/it]
  3% 491/18000 [1:16:50<39:21:30,  8.09s/it]
  3% 492/18000 [1:16:54<34:02:14,  7.00s/it]
  3% 493/18000 [1:17:11<48:38:09, 10.00s/it]
  3% 494/18000 [1:17:25<53:24:35, 10.98s/it]
  3% 495/18000 [1:17:35<52:53:14, 10.88s/it]
  3% 496/18000 [1:17:45<50:41:49, 10.43s/it]
  3% 497/18000 [1:17:53<47:21:35,  9.74s/it]
  3% 498/18000 [1:18:00<43:18:58,  8.91s/it]
  3% 499/18000 [1:18:05<38:28:50,  7.92s/it]
  3% 500/18000 [1:18:10<33:36:35,  6.91s/it]
                                            
  3% 500/18000 [1:18:10<33:36:35,  6.91s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 2609
+  Batch size = 80
+
+
  0% 0/33 [00:00<?, ?it/s]
+
  6% 2/33 [00:04<01:10,  2.26s/it]
+
  9% 3/33 [00:08<01:33,  3.13s/it]
+
 12% 4/33 [00:12<01:39,  3.44s/it]
+
 15% 5/33 [00:16<01:42,  3.64s/it]
+
 18% 6/33 [00:20<01:42,  3.79s/it]
+
 21% 7/33 [00:24<01:38,  3.80s/it]
+
 24% 8/33 [00:28<01:36,  3.85s/it]
+
 27% 9/33 [00:32<01:33,  3.88s/it]
+
 30% 10/33 [00:37<01:33,  4.07s/it]
+
 33% 11/33 [00:41<01:29,  4.06s/it]
+
 36% 12/33 [00:45<01:24,  4.04s/it]
+
 39% 13/33 [00:49<01:19,  3.98s/it]
+
 42% 14/33 [00:53<01:16,  4.03s/it]
+
 45% 15/33 [00:57<01:12,  4.04s/it]
+
 48% 16/33 [01:01<01:10,  4.17s/it]
+
 52% 17/33 [01:06<01:09,  4.32s/it]
+
 55% 18/33 [01:10<01:03,  4.24s/it]
+
 58% 19/33 [01:14<00:59,  4.24s/it]
+
 61% 20/33 [01:19<00:55,  4.31s/it]
+
 64% 21/33 [01:23<00:52,  4.34s/it]
+
 67% 22/33 [01:27<00:47,  4.33s/it]
+
 70% 23/33 [01:32<00:43,  4.38s/it]
+
 73% 24/33 [01:36<00:39,  4.39s/it]
+
 76% 25/33 [01:41<00:35,  4.42s/it]
+
 79% 26/33 [01:45<00:30,  4.37s/it]
+
 82% 27/33 [01:49<00:25,  4.31s/it]
+
 85% 28/33 [01:54<00:21,  4.34s/it]
+
 88% 29/33 [01:57<00:16,  4.09s/it]
+
 91% 30/33 [02:00<00:11,  3.70s/it]
+
 94% 31/33 [02:03<00:06,  3.40s/it]
+
 97% 32/33 [02:05<00:03,  3.18s/it]
+
+
                                   
                                            
+
  3% 500/18000 [1:20:23<33:36:35,  6.91s/it]
+
                                   Saving model checkpoint to ./checkpoint-500
+Configuration saved in ./checkpoint-500/config.json
+Model weights saved in ./checkpoint-500/pytorch_model.bin
+Configuration saved in ./checkpoint-500/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json

logs

@@ -0,0 +1,99 @@
+01/26/2022 10:56:56 - WARNING - __main__ - Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: True
+01/26/2022 10:56:56 - INFO - __main__ - Training/evaluation parameters TrainingArguments(
+_n_gpu=1,
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+bf16=False,
+bf16_full_eval=False,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_pin_memory=True,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+do_eval=True,
+do_predict=False,
+do_train=True,
+eval_accumulation_steps=None,
+eval_steps=500,
+evaluation_strategy=IntervalStrategy.STEPS,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+gradient_accumulation_steps=2,
+gradient_checkpointing=True,
+greater_is_better=None,
+group_by_length=True,
+half_precision_backend=auto,
+hub_model_id=None,
+hub_strategy=HubStrategy.EVERY_SAVE,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0003,
+length_column_name=input_length,
+load_best_model_at_end=False,
+local_rank=-1,
+log_level=-1,
+log_level_replica=-1,
+log_on_each_node=True,
+logging_dir=./runs/Jan26_10-56-56_ganymede.eafit.edu.co,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=IntervalStrategy.STEPS,
+lr_scheduler_type=SchedulerType.COSINE,
+max_grad_norm=1.0,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+no_cuda=False,
+num_train_epochs=500.0,
+optim=OptimizerNames.ADAMW_HF,
+output_dir=./,
+overwrite_output_dir=True,
+past_index=-1,
+per_device_eval_batch_size=80,
+per_device_train_batch_size=80,
+prediction_loss_only=False,
+push_to_hub=True,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+remove_unused_columns=True,
+report_to=['wandb'],
+resume_from_checkpoint=None,
+run_name=cosine+drop_proj+low_specaugment-300M,
+save_on_each_node=False,
+save_steps=500,
+save_strategy=IntervalStrategy.STEPS,
+save_total_limit=1,
+seed=42,
+sharded_ddp=[],
+skip_memory_metrics=True,
+tf32=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_legacy_prediction_loop=False,
+warmup_ratio=0.0,
+warmup_steps=500,
+weight_decay=0.0,
+xpu_backend=None,
+)
+01/26/2022 10:57:00 - WARNING - datasets.builder - Reusing dataset common_voice (/home/sagrilaft/.cache/huggingface/datasets/mozilla-foundation___common_voice/et/7.0.0/33e08856cfa0d0665e837bcad73ffd920a0bc713ce8c5fffb55dbdf1c084d5ba)
+01/26/2022 10:57:02 - WARNING - datasets.builder - Reusing dataset common_voice (/home/sagrilaft/.cache/huggingface/datasets/mozilla-foundation___common_voice/et/7.0.0/33e08856cfa0d0665e837bcad73ffd920a0bc713ce8c5fffb55dbdf1c084d5ba)
+01/26/2022 10:57:02 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/sagrilaft/.cache/huggingface/datasets/mozilla-foundation___common_voice/et/7.0.0/33e08856cfa0d0665e837bcad73ffd920a0bc713ce8c5fffb55dbdf1c084d5ba/cache-f4479ce33e07f5c7.arrow
+01/26/2022 10:57:02 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/sagrilaft/.cache/huggingface/datasets/mozilla-foundation___common_voice/et/7.0.0/33e08856cfa0d0665e837bcad73ffd920a0bc713ce8c5fffb55dbdf1c084d5ba/cache-105cefde64f772a9.arrow
+01/26/2022 10:58:44 - WARNING - huggingface_hub.repository - /home/sagrilaft/Project/audio/xls-r-et/./ is already a clone of https://huggingface.co/shpotes/xls-r-et. Make sure you pull the latest changes with `repo.git_pull()`.
+
+{'loss': 4.5903, 'learning_rate': 0.00019800000000000002, 'epoch': 2.78}
+{'loss': 2.9083, 'learning_rate': 0.000398, 'epoch': 5.56}
+{'loss': 1.2823, 'learning_rate': 0.000598, 'epoch': 8.33}
+{'loss': 0.3962, 'learning_rate': 0.0007980000000000001, 'epoch': 11.11}
+{'loss': 0.3034, 'learning_rate': 0.000998, 'epoch': 13.89}

preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aae5e50ef4a8b89fd5fb88e1636f3e95144e39211b78b0a3f3767d443d62958
+size 1262083569

run-1b.sh

@@ -13,7 +13,7 @@ python src/run_speech_recognition_ctc_bnb.py \
        --per_device_train_batch_size=48 \
        --per_device_eval_batch_size=48 \
        --gradient_accumulation_steps=2 \
-       --learning_rate=3e-4 \
+       --learning_rate=5e-5 \
        --save_total_limit=1 \
        --warmup_steps=500 \
        --evaluation_strategy=steps \

run-300M.sh

@@ -9,7 +9,7 @@ python src/run_speech_recognition_ctc_bnb.py \
        --dataset_config_name="et" \
        --output_dir="./" \
        --overwrite_output_dir \
-       --num_train_epochs=100 \
+       --num_train_epochs=500 \
        --per_device_train_batch_size=80 \
        --per_device_eval_batch_size=80 \
        --gradient_accumulation_steps=2 \
@@ -19,8 +19,8 @@ python src/run_speech_recognition_ctc_bnb.py \
        --evaluation_strategy=steps \
        --text_column_name=sentence \
        --length_column_name=input_length \
-       --save_steps=1000 \
-       --eval_steps=250 \
+       --save_steps=500 \
+       --eval_steps=500 \
        --logging_steps=100 \
        --layerdrop=0.0 \
        --freeze_feature_encoder \

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9be3a7c88f8500cf6b3a8b0b089125768d465147886310e4abb4ddc8cf154dd1
+size 3055

vocab.json

@@ -0,0 +1 @@
+{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, "h": 8, "i": 9, "j": 10, "k": 11, "l": 12, "m": 13, "n": 14, "o": 15, "p": 16, "q": 17, "r": 18, "s": 19, "t": 20, "u": 21, "v": 22, "w": 23, "x": 24, "y": 25, "z": 26, "ä": 27, "õ": 28, "ö": 29, "ü": 30, "š": 31, "ž": 32, "̇": 33, "„": 34, "|": 0, "[UNK]": 35, "[PAD]": 36}

wandb/debug-internal.log

@@ -0,0 +1 @@
+run-20220126_121554-295enqix/logs/debug-internal.log

wandb/debug.log

@@ -0,0 +1 @@
+run-20220126_121554-295enqix/logs/debug.log

wandb/latest-run

@@ -0,0 +1 @@
+run-20220126_121554-295enqix

wandb/run-20220126_105847-1xdiy2kf/files/output.log

@@ -0,0 +1,544 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3% 500/18000 [1:18:10<33:36:35,  6.91s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 2609
+  Batch size = 80
+{'loss': 4.5903, 'learning_rate': 0.00019800000000000002, 'epoch': 2.78}
+{'loss': 2.9083, 'learning_rate': 0.000398, 'epoch': 5.56}
+{'loss': 1.2823, 'learning_rate': 0.000598, 'epoch': 8.33}
+{'loss': 0.3962, 'learning_rate': 0.0007980000000000001, 'epoch': 11.11}
+{'loss': 0.3034, 'learning_rate': 0.000998, 'epoch': 13.89}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Configuration saved in ./checkpoint-500/config.json
+Model weights saved in ./checkpoint-500/pytorch_model.bin
+Configuration saved in ./checkpoint-500/preprocessor_config.json

wandb/run-20220126_105847-1xdiy2kf/files/requirements.txt

@@ -0,0 +1,87 @@
+aiohttp==3.8.1
+aiosignal==1.2.0
+appdirs==1.4.4
+async-timeout==4.0.2
+attrs==21.4.0
+audioread==2.1.9
+bitsandbytes-cuda113==0.26.0
+certifi==2021.10.8
+cffi==1.15.0
+charset-normalizer==2.0.10
+click==8.0.3
+clldutils==3.10.1
+colorlog==6.6.0
+configparser==5.2.0
+csvw==1.11.0
+datasets==1.18.1.dev0
+decorator==5.1.1
+dill==0.3.4
+dlinfo==1.2.1
+docker-pycreds==0.4.0
+filelock==3.4.2
+frozenlist==1.3.0
+fsspec==2022.1.0
+gitdb==4.0.9
+gitpython==3.1.26
+huggingface-hub==0.4.0
+hypothesis==6.36.0
+idna==3.3
+isodate==0.6.1
+jiwer==2.3.0
+joblib==1.1.0
+librosa==0.8.1
+llvmlite==0.38.0
+multidict==6.0.2
+multiprocess==0.70.12.2
+numba==0.55.0
+numpy==1.21.5
+packaging==21.3
+pandas==1.4.0
+pathtools==0.1.2
+phonemizer==3.0.1
+pip==21.3.1
+pooch==1.6.0
+promise==2.3
+protobuf==3.19.3
+psutil==5.9.0
+pyarrow==6.0.1
+pycparser==2.21
+pyctcdecode==0.3.0
+pygtrie==2.4.2
+pyparsing==3.0.7
+python-dateutil==2.8.2
+python-levenshtein==0.12.2
+pytz==2021.3
+pyyaml==6.0
+regex==2022.1.18
+requests==2.27.1
+resampy==0.2.2
+rfc3986==2.0.0
+sacremoses==0.0.47
+scikit-learn==1.0.2
+scipy==1.7.3
+segments==2.2.0
+sentry-sdk==1.5.4
+setuptools==60.2.0
+shortuuid==1.0.8
+six==1.16.0
+smmap==5.0.0
+sortedcontainers==2.4.0
+soundfile==0.10.3.post1
+subprocess32==3.5.4
+tabulate==0.8.9
+termcolor==1.1.0
+threadpoolctl==3.0.0
+tokenizers==0.11.4
+torch==1.10.1
+torchaudio==0.10.1
+tqdm==4.62.3
+transformers==4.16.0.dev0
+typing-extensions==4.0.1
+uritemplate==4.1.1
+urllib3==1.26.8
+wandb==0.12.9
+wheel==0.37.1
+xxhash==2.0.2
+yarl==1.7.2
+yaspin==2.1.0

wandb/run-20220126_105847-1xdiy2kf/files/wandb-metadata.json

@@ -0,0 +1,76 @@
+{
+    "os": "Linux-4.18.0-305.10.2.el8_4.x86_64-x86_64-with-glibc2.28",
+    "python": "3.9.6",
+    "heartbeatAt": "2022-01-26T15:58:48.495262",
+    "startedAt": "2022-01-26T15:58:47.536839",
+    "docker": null,
+    "gpu": "Tesla V100-PCIE-32GB",
+    "gpu_count": 3,
+    "cpu_count": 64,
+    "cuda": null,
+    "args": [
+        "--dataset_name=mozilla-foundation/common_voice_7_0",
+        "--model_name_or_path=facebook/wav2vec2-xls-r-300m",
+        "--dataset_config_name=et",
+        "--output_dir=./",
+        "--overwrite_output_dir",
+        "--num_train_epochs=500",
+        "--per_device_train_batch_size=80",
+        "--per_device_eval_batch_size=80",
+        "--gradient_accumulation_steps=2",
+        "--learning_rate=3e-4",
+        "--save_total_limit=1",
+        "--warmup_steps=500",
+        "--evaluation_strategy=steps",
+        "--text_column_name=sentence",
+        "--length_column_name=input_length",
+        "--save_steps=500",
+        "--eval_steps=500",
+        "--logging_steps=100",
+        "--layerdrop=0.0",
+        "--freeze_feature_encoder",
+        "--feat_proj_dropout=0.1",
+        "--chars_to_ignore",
+        ",",
+        "?",
+        ".",
+        "!",
+        "-",
+        ";",
+        ":",
+        "\"",
+        "\u201c",
+        "%",
+        "\u2018",
+        "\u201d",
+        "\ufffd",
+        "\u2014",
+        "\u2019",
+        "\u2026",
+        "\u2013",
+        "--gradient_checkpointing",
+        "--lr_scheduler_type=cosine",
+        "--fp16",
+        "--group_by_length",
+        "--mask_time_prob=0.1",
+        "--mask_time_length=10",
+        "--report_to=wandb",
+        "--run_name=cosine+drop_proj+low_specaugment-300M",
+        "--do_train",
+        "--do_eval",
+        "--use_auth_token",
+        "--push_to_hub"
+    ],
+    "state": "running",
+    "program": "/home/sagrilaft/Project/audio/xls-r-et/src/run_speech_recognition_ctc_bnb.py",
+    "codePath": "src/run_speech_recognition_ctc_bnb.py",
+    "git": {
+        "remote": "https://huggingface.co/shpotes/xls-r-et",
+        "commit": "2a437db2644553c5c9c2b9e81553ed9bdf92207e"
+    },
+    "email": "shpotes3@gmail.com",
+    "root": "/home/sagrilaft/Project/audio/xls-r-et",
+    "host": "ganymede.eafit.edu.co",
+    "username": "sagrilaft",
+    "executable": "/home/sagrilaft/Project/audio/xls-r-et/.venv/bin/python"
+}

wandb/run-20220126_105847-1xdiy2kf/logs/debug.log

@@ -0,0 +1,24 @@
+2022-01-26 10:58:47,539 INFO    MainThread:2220463 [wandb_setup.py:_flush():71] setting env: {'project': 'xls-r-estonian'}
+2022-01-26 10:58:47,540 INFO    MainThread:2220463 [wandb_setup.py:_flush():71] setting login settings: {}
+2022-01-26 10:58:47,540 INFO    MainThread:2220463 [wandb_init.py:_log_setup():371] Logging user logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_105847-1xdiy2kf/logs/debug.log
+2022-01-26 10:58:47,540 INFO    MainThread:2220463 [wandb_init.py:_log_setup():372] Logging internal logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_105847-1xdiy2kf/logs/debug-internal.log
+2022-01-26 10:58:47,540 INFO    MainThread:2220463 [wandb_init.py:init():404] calling init triggers
+2022-01-26 10:58:47,541 INFO    MainThread:2220463 [wandb_init.py:init():409] wandb.init called with sweep_config: {}
+config: {}
+2022-01-26 10:58:47,541 INFO    MainThread:2220463 [wandb_init.py:init():460] starting backend
+2022-01-26 10:58:47,541 INFO    MainThread:2220463 [backend.py:_multiprocessing_setup():99] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2022-01-26 10:58:47,575 INFO    MainThread:2220463 [backend.py:ensure_launched():216] starting backend process...
+2022-01-26 10:58:47,598 INFO    MainThread:2220463 [backend.py:ensure_launched():221] started backend process with pid: 2220828
+2022-01-26 10:58:47,600 INFO    MainThread:2220463 [wandb_init.py:init():469] backend started and connected
+2022-01-26 10:58:47,604 INFO    MainThread:2220463 [wandb_init.py:init():533] updated telemetry
+2022-01-26 10:58:47,664 INFO    MainThread:2220463 [wandb_init.py:init():563] communicating current version
+2022-01-26 10:58:48,298 INFO    MainThread:2220463 [wandb_init.py:init():568] got version response 
+2022-01-26 10:58:48,299 INFO    MainThread:2220463 [wandb_init.py:init():578] communicating run to backend with 30 second timeout
+2022-01-26 10:58:48,488 INFO    MainThread:2220463 [wandb_init.py:init():606] starting run threads in backend
+2022-01-26 10:58:48,534 INFO    MainThread:2220463 [wandb_run.py:_console_start():1810] atexit reg
+2022-01-26 10:58:48,535 INFO    MainThread:2220463 [wandb_run.py:_redirect():1684] redirect: SettingsConsole.REDIRECT
+2022-01-26 10:58:48,536 INFO    MainThread:2220463 [wandb_run.py:_redirect():1689] Redirecting console.
+2022-01-26 10:58:48,541 INFO    MainThread:2220463 [wandb_run.py:_redirect():1745] Redirects installed.
+2022-01-26 10:58:48,541 INFO    MainThread:2220463 [wandb_init.py:init():633] run started, returning control to user process
+2022-01-26 10:58:48,562 INFO    MainThread:2220463 [wandb_run.py:_config_callback():956] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'architectures': ['Wav2Vec2ForPreTraining'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': 36, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'facebook/wav2vec2-xls-r-300m', 'transformers_version': '4.16.0.dev0', 'feat_extract_dropout': 0.0, 'model_type': 'wav2vec2', 'num_feat_extract_layers': 7, 'hidden_size': 1024, 'feat_extract_norm': 'layer', 'feat_extract_activation': 'gelu', 'conv_dim': [512, 512, 512, 512, 512, 512, 512], 'conv_stride': [5, 2, 2, 2, 2, 2, 2], 'conv_kernel': [10, 3, 3, 3, 3, 2, 2], 'conv_bias': True, 'num_conv_pos_embeddings': 128, 'num_conv_pos_embedding_groups': 16, 'num_hidden_layers': 24, 'intermediate_size': 4096, 'hidden_act': 'gelu', 'num_attention_heads': 16, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'activation_dropout': 0.0, 'feat_proj_dropout': 0.1, 'final_dropout': 0.0, 'layerdrop': 0.0, 'layer_norm_eps': 1e-05, 'initializer_range': 0.02, 'vocab_size': 39, 'do_stable_layer_norm': True, 'use_weighted_layer_sum': False, 'apply_spec_augment': True, 'mask_time_prob': 0.1, 'mask_time_length': 10, 'mask_time_min_masks': 2, 'mask_feature_prob': 0.0, 'mask_feature_length': 10, 'mask_feature_min_masks': 0, 'num_codevectors_per_group': 320, 'num_codevector_groups': 2, 'contrastive_logits_temperature': 0.1, 'feat_quantizer_dropout': 0.0, 'num_negatives': 100, 'codevector_dim': 768, 'proj_codevector_dim': 768, 'diversity_loss_weight': 0.1, 'ctc_loss_reduction': 'mean', 'ctc_zero_infinity': False, 'add_adapter': False, 'adapter_kernel_size': 3, 'adapter_stride': 2, 'num_adapter_layers': 3, 'output_hidden_size': 1024, 'classifier_proj_size': 256, 'tdnn_dim': [512, 512, 512, 512, 1500], 'tdnn_kernel': [5, 3, 3, 1, 1], 'tdnn_dilation': [1, 2, 3, 1, 1], 'xvector_output_dim': 512, 'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 80, 'per_device_eval_batch_size': 80, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': 'None', 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 500.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.0, 'warmup_steps': 500, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jan26_10-56-56_ganymede.eafit.edu.co', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'amp', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': -1, 'xpu_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'cosine+drop_proj+low_specaugment-300M', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_hf', 'adafactor': False, 'group_by_length': True, 'length_column_name': 'input_length', 'report_to': "['wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'None', 'hub_model_id': 'None', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'gradient_checkpointing': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', '_n_gpu': 1, 'mp_parameters': '', 'train_batch_size': 80, 'eval_batch_size': 80}
+2022-01-26 10:58:48,565 INFO    MainThread:2220463 [wandb_watch.py:watch():43] Watching

wandb/run-20220126_121554-295enqix/files/config.yaml

@@ -0,0 +1,649 @@
+wandb_version: 1
+
+_n_gpu:
+  desc: null
+  value: 1
+_name_or_path:
+  desc: null
+  value: facebook/wav2vec2-xls-r-1b
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.12.9
+    framework: huggingface
+    huggingface_version: 4.16.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: true
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+    python_version: 3.9.6
+    start_time: 1643217354
+    t:
+      1:
+      - 1
+      - 5
+      - 11
+      3:
+      - 13
+      4: 3.9.6
+      5: 0.12.9
+      6: 4.16.0.dev0
+      8:
+      - 2
+      - 5
+activation_dropout:
+  desc: null
+  value: 0.0
+adafactor:
+  desc: null
+  value: false
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+adapter_kernel_size:
+  desc: null
+  value: 3
+adapter_stride:
+  desc: null
+  value: 2
+add_adapter:
+  desc: null
+  value: false
+add_cross_attention:
+  desc: null
+  value: false
+apply_spec_augment:
+  desc: null
+  value: true
+architectures:
+  desc: null
+  value:
+  - Wav2Vec2ForPreTraining
+attention_dropout:
+  desc: null
+  value: 0.0
+bad_words_ids:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: false
+bf16_full_eval:
+  desc: null
+  value: false
+bos_token_id:
+  desc: null
+  value: 1
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+classifier_proj_size:
+  desc: null
+  value: 256
+codevector_dim:
+  desc: null
+  value: 1024
+contrastive_logits_temperature:
+  desc: null
+  value: 0.1
+conv_bias:
+  desc: null
+  value: true
+conv_dim:
+  desc: null
+  value:
+  - 512
+  - 512
+  - 512
+  - 512
+  - 512
+  - 512
+  - 512
+conv_kernel:
+  desc: null
+  value:
+  - 10
+  - 3
+  - 3
+  - 3
+  - 3
+  - 2
+  - 2
+conv_stride:
+  desc: null
+  value:
+  - 5
+  - 2
+  - 2
+  - 2
+  - 2
+  - 2
+  - 2
+cross_attention_hidden_size:
+  desc: null
+  value: null
+ctc_loss_reduction:
+  desc: null
+  value: mean
+ctc_zero_infinity:
+  desc: null
+  value: false
+dataloader_drop_last:
+  desc: null
+  value: false
+dataloader_num_workers:
+  desc: null
+  value: 0
+dataloader_pin_memory:
+  desc: null
+  value: true
+ddp_bucket_cap_mb:
+  desc: null
+  value: None
+ddp_find_unused_parameters:
+  desc: null
+  value: None
+debug:
+  desc: null
+  value: '[]'
+decoder_start_token_id:
+  desc: null
+  value: null
+deepspeed:
+  desc: null
+  value: None
+disable_tqdm:
+  desc: null
+  value: false
+diversity_loss_weight:
+  desc: null
+  value: 0.1
+diversity_penalty:
+  desc: null
+  value: 0.0
+do_eval:
+  desc: null
+  value: true
+do_predict:
+  desc: null
+  value: false
+do_sample:
+  desc: null
+  value: false
+do_stable_layer_norm:
+  desc: null
+  value: true
+do_train:
+  desc: null
+  value: true
+early_stopping:
+  desc: null
+  value: false
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+eos_token_id:
+  desc: null
+  value: 2
+eval_accumulation_steps:
+  desc: null
+  value: None
+eval_batch_size:
+  desc: null
+  value: 48
+eval_steps:
+  desc: null
+  value: 250
+evaluation_strategy:
+  desc: null
+  value: steps
+feat_extract_activation:
+  desc: null
+  value: gelu
+feat_extract_dropout:
+  desc: null
+  value: 0.0
+feat_extract_norm:
+  desc: null
+  value: layer
+feat_proj_dropout:
+  desc: null
+  value: 0.1
+feat_quantizer_dropout:
+  desc: null
+  value: 0.0
+final_dropout:
+  desc: null
+  value: 0.0
+finetuning_task:
+  desc: null
+  value: null
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+fp16:
+  desc: null
+  value: true
+fp16_backend:
+  desc: null
+  value: auto
+fp16_full_eval:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+gradient_accumulation_steps:
+  desc: null
+  value: 2
+gradient_checkpointing:
+  desc: null
+  value: true
+greater_is_better:
+  desc: null
+  value: None
+group_by_length:
+  desc: null
+  value: true
+half_precision_backend:
+  desc: null
+  value: amp
+hidden_act:
+  desc: null
+  value: gelu
+hidden_dropout:
+  desc: null
+  value: 0.0
+hidden_size:
+  desc: null
+  value: 1280
+hub_model_id:
+  desc: null
+  value: None
+hub_strategy:
+  desc: null
+  value: every_save
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+ignore_data_skip:
+  desc: null
+  value: false
+initializer_range:
+  desc: null
+  value: 0.02
+intermediate_size:
+  desc: null
+  value: 5120
+is_decoder:
+  desc: null
+  value: false
+is_encoder_decoder:
+  desc: null
+  value: false
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+label_names:
+  desc: null
+  value: None
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+layer_norm_eps:
+  desc: null
+  value: 1.0e-05
+layerdrop:
+  desc: null
+  value: 0.0
+learning_rate:
+  desc: null
+  value: 5.0e-05
+length_column_name:
+  desc: null
+  value: input_length
+length_penalty:
+  desc: null
+  value: 1.0
+load_best_model_at_end:
+  desc: null
+  value: false
+local_rank:
+  desc: null
+  value: -1
+log_level:
+  desc: null
+  value: -1
+log_level_replica:
+  desc: null
+  value: -1
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./1B/runs/Jan26_12-15-06_ganymede.eafit.edu.co
+logging_first_step:
+  desc: null
+  value: false
+logging_nan_inf_filter:
+  desc: null
+  value: true
+logging_steps:
+  desc: null
+  value: 100
+logging_strategy:
+  desc: null
+  value: steps
+lr_scheduler_type:
+  desc: null
+  value: cosine
+mask_feature_length:
+  desc: null
+  value: 10
+mask_feature_min_masks:
+  desc: null
+  value: 0
+mask_feature_prob:
+  desc: null
+  value: 0.0
+mask_time_length:
+  desc: null
+  value: 10
+mask_time_min_masks:
+  desc: null
+  value: 2
+mask_time_prob:
+  desc: null
+  value: 0.1
+max_grad_norm:
+  desc: null
+  value: 1.0
+max_length:
+  desc: null
+  value: 20
+max_steps:
+  desc: null
+  value: -1
+metric_for_best_model:
+  desc: null
+  value: None
+min_length:
+  desc: null
+  value: 0
+model_type:
+  desc: null
+  value: wav2vec2
+mp_parameters:
+  desc: null
+  value: ''
+no_cuda:
+  desc: null
+  value: false
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+num_adapter_layers:
+  desc: null
+  value: 3
+num_attention_heads:
+  desc: null
+  value: 16
+num_beam_groups:
+  desc: null
+  value: 1
+num_beams:
+  desc: null
+  value: 1
+num_codevector_groups:
+  desc: null
+  value: 2
+num_codevectors_per_group:
+  desc: null
+  value: 320
+num_conv_pos_embedding_groups:
+  desc: null
+  value: 16
+num_conv_pos_embeddings:
+  desc: null
+  value: 128
+num_feat_extract_layers:
+  desc: null
+  value: 7
+num_hidden_layers:
+  desc: null
+  value: 48
+num_negatives:
+  desc: null
+  value: 100
+num_return_sequences:
+  desc: null
+  value: 1
+num_train_epochs:
+  desc: null
+  value: 100.0
+optim:
+  desc: null
+  value: adamw_hf
+output_attentions:
+  desc: null
+  value: false
+output_dir:
+  desc: null
+  value: ./1B
+output_hidden_size:
+  desc: null
+  value: 1280
+output_hidden_states:
+  desc: null
+  value: false
+output_scores:
+  desc: null
+  value: false
+overwrite_output_dir:
+  desc: null
+  value: true
+pad_token_id:
+  desc: null
+  value: 36
+past_index:
+  desc: null
+  value: -1
+per_device_eval_batch_size:
+  desc: null
+  value: 48
+per_device_train_batch_size:
+  desc: null
+  value: 48
+per_gpu_eval_batch_size:
+  desc: null
+  value: None
+per_gpu_train_batch_size:
+  desc: null
+  value: None
+prediction_loss_only:
+  desc: null
+  value: false
+prefix:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+proj_codevector_dim:
+  desc: null
+  value: 1024
+pruned_heads:
+  desc: null
+  value: {}
+push_to_hub:
+  desc: null
+  value: false
+push_to_hub_model_id:
+  desc: null
+  value: None
+push_to_hub_organization:
+  desc: null
+  value: None
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+remove_invalid_values:
+  desc: null
+  value: false
+remove_unused_columns:
+  desc: null
+  value: true
+repetition_penalty:
+  desc: null
+  value: 1.0
+report_to:
+  desc: null
+  value: '[''wandb'']'
+resume_from_checkpoint:
+  desc: null
+  value: None
+return_dict:
+  desc: null
+  value: true
+return_dict_in_generate:
+  desc: null
+  value: false
+run_name:
+  desc: null
+  value: cosine+drop_proj+low_specaugment-1b
+save_on_each_node:
+  desc: null
+  value: false
+save_steps:
+  desc: null
+  value: 1000
+save_strategy:
+  desc: null
+  value: steps
+save_total_limit:
+  desc: null
+  value: 1
+seed:
+  desc: null
+  value: 42
+sep_token_id:
+  desc: null
+  value: null
+sharded_ddp:
+  desc: null
+  value: '[]'
+skip_memory_metrics:
+  desc: null
+  value: true
+task_specific_params:
+  desc: null
+  value: null
+tdnn_dilation:
+  desc: null
+  value:
+  - 1
+  - 2
+  - 3
+  - 1
+  - 1
+tdnn_dim:
+  desc: null
+  value:
+  - 512
+  - 512
+  - 512
+  - 512
+  - 1500
+tdnn_kernel:
+  desc: null
+  value:
+  - 5
+  - 3
+  - 3
+  - 1
+  - 1
+temperature:
+  desc: null
+  value: 1.0
+tf32:
+  desc: null
+  value: None
+tie_encoder_decoder:
+  desc: null
+  value: false
+tie_word_embeddings:
+  desc: null
+  value: true
+tokenizer_class:
+  desc: null
+  value: null
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+torch_dtype:
+  desc: null
+  value: float32
+torchscript:
+  desc: null
+  value: false
+tpu_metrics_debug:
+  desc: null
+  value: false
+tpu_num_cores:
+  desc: null
+  value: None
+train_batch_size:
+  desc: null
+  value: 48
+transformers_version:
+  desc: null
+  value: 4.16.0.dev0
+use_bfloat16:
+  desc: null
+  value: false
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+use_weighted_layer_sum:
+  desc: null
+  value: false
+vocab_size:
+  desc: null
+  value: 39
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 500
+weight_decay:
+  desc: null
+  value: 0.0
+xpu_backend:
+  desc: null
+  value: None
+xvector_output_dim:
+  desc: null
+  value: 512

wandb/run-20220126_121554-295enqix/files/output.log

@@ -0,0 +1,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

wandb/run-20220126_121554-295enqix/files/requirements.txt

@@ -0,0 +1,87 @@
+aiohttp==3.8.1
+aiosignal==1.2.0
+appdirs==1.4.4
+async-timeout==4.0.2
+attrs==21.4.0
+audioread==2.1.9
+bitsandbytes-cuda113==0.26.0
+certifi==2021.10.8
+cffi==1.15.0
+charset-normalizer==2.0.10
+click==8.0.3
+clldutils==3.10.1
+colorlog==6.6.0
+configparser==5.2.0
+csvw==1.11.0
+datasets==1.18.1.dev0
+decorator==5.1.1
+dill==0.3.4
+dlinfo==1.2.1
+docker-pycreds==0.4.0
+filelock==3.4.2
+frozenlist==1.3.0
+fsspec==2022.1.0
+gitdb==4.0.9
+gitpython==3.1.26
+huggingface-hub==0.4.0
+hypothesis==6.36.0
+idna==3.3
+isodate==0.6.1
+jiwer==2.3.0
+joblib==1.1.0
+librosa==0.8.1
+llvmlite==0.38.0
+multidict==6.0.2
+multiprocess==0.70.12.2
+numba==0.55.0
+numpy==1.21.5
+packaging==21.3
+pandas==1.4.0
+pathtools==0.1.2
+phonemizer==3.0.1
+pip==21.3.1
+pooch==1.6.0
+promise==2.3
+protobuf==3.19.3
+psutil==5.9.0
+pyarrow==6.0.1
+pycparser==2.21
+pyctcdecode==0.3.0
+pygtrie==2.4.2
+pyparsing==3.0.7
+python-dateutil==2.8.2
+python-levenshtein==0.12.2
+pytz==2021.3
+pyyaml==6.0
+regex==2022.1.18
+requests==2.27.1
+resampy==0.2.2
+rfc3986==2.0.0
+sacremoses==0.0.47
+scikit-learn==1.0.2
+scipy==1.7.3
+segments==2.2.0
+sentry-sdk==1.5.4
+setuptools==60.2.0
+shortuuid==1.0.8
+six==1.16.0
+smmap==5.0.0
+sortedcontainers==2.4.0
+soundfile==0.10.3.post1
+subprocess32==3.5.4
+tabulate==0.8.9
+termcolor==1.1.0
+threadpoolctl==3.0.0
+tokenizers==0.11.4
+torch==1.10.1
+torchaudio==0.10.1
+tqdm==4.62.3
+transformers==4.16.0.dev0
+typing-extensions==4.0.1
+uritemplate==4.1.1
+urllib3==1.26.8
+wandb==0.12.9
+wheel==0.37.1
+xxhash==2.0.2
+yarl==1.7.2
+yaspin==2.1.0

wandb/run-20220126_121554-295enqix/files/wandb-metadata.json

@@ -0,0 +1,74 @@
+{
+    "os": "Linux-4.18.0-305.10.2.el8_4.x86_64-x86_64-with-glibc2.28",
+    "python": "3.9.6",
+    "heartbeatAt": "2022-01-26T17:15:55.285233",
+    "startedAt": "2022-01-26T17:15:54.341820",
+    "docker": null,
+    "gpu": "Tesla V100-PCIE-32GB",
+    "gpu_count": 3,
+    "cpu_count": 64,
+    "cuda": null,
+    "args": [
+        "--dataset_name=mozilla-foundation/common_voice_7_0",
+        "--model_name_or_path=facebook/wav2vec2-xls-r-1b",
+        "--dataset_config_name=et",
+        "--output_dir=./1B",
+        "--overwrite_output_dir",
+        "--num_train_epochs=100",
+        "--per_device_train_batch_size=48",
+        "--per_device_eval_batch_size=48",
+        "--gradient_accumulation_steps=2",
+        "--learning_rate=5e-5",
+        "--save_total_limit=1",
+        "--warmup_steps=500",
+        "--evaluation_strategy=steps",
+        "--text_column_name=sentence",
+        "--length_column_name=input_length",
+        "--save_steps=1000",
+        "--eval_steps=250",
+        "--logging_steps=100",
+        "--layerdrop=0.0",
+        "--freeze_feature_encoder",
+        "--feat_proj_dropout=0.1",
+        "--chars_to_ignore",
+        ",",
+        "?",
+        ".",
+        "!",
+        "-",
+        ";",
+        ":",
+        "\"",
+        "\u201c",
+        "%",
+        "\u2018",
+        "\u201d",
+        "\ufffd",
+        "\u2014",
+        "\u2019",
+        "\u2026",
+        "\u2013",
+        "--gradient_checkpointing",
+        "--lr_scheduler_type=cosine",
+        "--fp16",
+        "--group_by_length",
+        "--mask_time_prob=0.1",
+        "--mask_time_length=10",
+        "--report_to=wandb",
+        "--run_name=cosine+drop_proj+low_specaugment-1b",
+        "--do_train",
+        "--do_eval"
+    ],
+    "state": "running",
+    "program": "/home/sagrilaft/Project/audio/xls-r-et/src/run_speech_recognition_ctc_bnb.py",
+    "codePath": "src/run_speech_recognition_ctc_bnb.py",
+    "git": {
+        "remote": "https://huggingface.co/shpotes/xls-r-et",
+        "commit": "2a437db2644553c5c9c2b9e81553ed9bdf92207e"
+    },
+    "email": "shpotes3@gmail.com",
+    "root": "/home/sagrilaft/Project/audio/xls-r-et",
+    "host": "ganymede.eafit.edu.co",
+    "username": "sagrilaft",
+    "executable": "/home/sagrilaft/Project/audio/xls-r-et/.venv/bin/python"
+}

wandb/run-20220126_121554-295enqix/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

wandb/run-20220126_121554-295enqix/logs/debug-internal.log

@@ -0,0 +1,87 @@
+2022-01-26 12:15:54,828 INFO    MainThread:2226380 [internal.py:wandb_internal():87] W&B internal server running at pid: 2226380, started at: 2022-01-26 12:15:54.828545
+2022-01-26 12:15:54,831 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: check_version
+2022-01-26 12:15:54,831 INFO    WriterThread:2226380 [datastore.py:open_for_write():77] open: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/run-295enqix.wandb
+2022-01-26 12:15:54,833 DEBUG   SenderThread:2226380 [sender.py:send():234] send: header
+2022-01-26 12:15:54,834 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: check_version
+2022-01-26 12:15:55,103 DEBUG   SenderThread:2226380 [sender.py:send():234] send: run
+2022-01-26 12:15:55,276 INFO    SenderThread:2226380 [dir_watcher.py:__init__():169] watching files in: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files
+2022-01-26 12:15:55,276 INFO    SenderThread:2226380 [sender.py:_start_run_threads():804] run started: 295enqix with start time 1643217354
+2022-01-26 12:15:55,276 DEBUG   SenderThread:2226380 [sender.py:send():234] send: summary
+2022-01-26 12:15:55,276 INFO    SenderThread:2226380 [sender.py:_save_file():939] saving file wandb-summary.json with policy end
+2022-01-26 12:15:55,278 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: run_start
+2022-01-26 12:15:55,284 DEBUG   HandlerThread:2226380 [meta.py:__init__():40] meta init
+2022-01-26 12:15:55,285 DEBUG   HandlerThread:2226380 [meta.py:__init__():54] meta init done
+2022-01-26 12:15:55,285 DEBUG   HandlerThread:2226380 [meta.py:probe():214] probe
+2022-01-26 12:15:55,291 DEBUG   HandlerThread:2226380 [meta.py:_setup_git():204] setup git
+2022-01-26 12:15:55,309 DEBUG   HandlerThread:2226380 [meta.py:_setup_git():211] setup git done
+2022-01-26 12:15:55,309 DEBUG   HandlerThread:2226380 [meta.py:_save_pip():58] save pip
+2022-01-26 12:15:55,309 DEBUG   HandlerThread:2226380 [meta.py:_save_pip():72] save pip done
+2022-01-26 12:15:55,309 DEBUG   HandlerThread:2226380 [meta.py:probe():252] probe done
+2022-01-26 12:15:55,313 DEBUG   SenderThread:2226380 [sender.py:send():234] send: files
+2022-01-26 12:15:55,313 INFO    SenderThread:2226380 [sender.py:_save_file():939] saving file wandb-metadata.json with policy now
+2022-01-26 12:15:55,322 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:15:55,323 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:15:55,460 DEBUG   SenderThread:2226380 [sender.py:send():234] send: config
+2022-01-26 12:15:55,461 DEBUG   SenderThread:2226380 [sender.py:send():234] send: metric
+2022-01-26 12:15:55,462 DEBUG   SenderThread:2226380 [sender.py:send():234] send: metric
+2022-01-26 12:15:55,462 WARNING SenderThread:2226380 [sender.py:send_metric():897] Seen metric with glob (shouldnt happen)
+2022-01-26 12:15:55,937 INFO    Thread-11 :2226380 [upload_job.py:push():137] Uploaded file /tmp/tmp0kjyvd1nwandb/sa83kpbt-wandb-metadata.json
+2022-01-26 12:15:56,277 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_created():217] file/dir created: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/wandb-metadata.json
+2022-01-26 12:15:56,278 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_created():217] file/dir created: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/requirements.txt
+2022-01-26 12:15:56,278 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_created():217] file/dir created: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:15:56,278 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_created():217] file/dir created: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/wandb-summary.json
+2022-01-26 12:15:58,277 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:16:10,577 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:16:10,577 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:16:16,285 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:16:24,276 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
+2022-01-26 12:16:25,718 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:16:25,719 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:16:26,289 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/config.yaml
+2022-01-26 12:16:30,291 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:16:40,861 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:16:40,862 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:16:44,298 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:16:55,255 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
+2022-01-26 12:16:56,119 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:16:56,119 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:16:56,303 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:17:06,308 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:17:11,395 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:17:11,395 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:17:16,312 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:17:26,266 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
+2022-01-26 12:17:26,317 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:17:26,631 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:17:26,632 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:17:34,321 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:17:41,857 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:17:41,858 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:17:42,325 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:17:50,329 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:17:56,331 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:17:57,090 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:17:57,090 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:17:57,233 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
+2022-01-26 12:18:02,334 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:18:08,337 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:18:12,358 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:18:12,359 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:18:13,339 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:18:27,555 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:18:27,555 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:18:28,215 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
+2022-01-26 12:18:31,347 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:18:42,695 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:18:42,695 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:18:45,354 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:18:57,952 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:18:57,952 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:18:59,199 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
+2022-01-26 12:18:59,360 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:19:11,366 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:19:13,205 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:19:13,206 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:19:23,373 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:19:28,370 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:19:28,371 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status

wandb/run-20220126_121554-295enqix/logs/debug.log

@@ -0,0 +1,24 @@
+2022-01-26 12:15:54,344 INFO    MainThread:2226158 [wandb_setup.py:_flush():71] setting env: {'project': 'xls-r-estonian'}
+2022-01-26 12:15:54,344 INFO    MainThread:2226158 [wandb_setup.py:_flush():71] setting login settings: {}
+2022-01-26 12:15:54,345 INFO    MainThread:2226158 [wandb_init.py:_log_setup():371] Logging user logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/logs/debug.log
+2022-01-26 12:15:54,345 INFO    MainThread:2226158 [wandb_init.py:_log_setup():372] Logging internal logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/logs/debug-internal.log
+2022-01-26 12:15:54,345 INFO    MainThread:2226158 [wandb_init.py:init():404] calling init triggers
+2022-01-26 12:15:54,346 INFO    MainThread:2226158 [wandb_init.py:init():409] wandb.init called with sweep_config: {}
+config: {}
+2022-01-26 12:15:54,346 INFO    MainThread:2226158 [wandb_init.py:init():460] starting backend
+2022-01-26 12:15:54,346 INFO    MainThread:2226158 [backend.py:_multiprocessing_setup():99] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2022-01-26 12:15:54,373 INFO    MainThread:2226158 [backend.py:ensure_launched():216] starting backend process...
+2022-01-26 12:15:54,395 INFO    MainThread:2226158 [backend.py:ensure_launched():221] started backend process with pid: 2226380
+2022-01-26 12:15:54,396 INFO    MainThread:2226158 [wandb_init.py:init():469] backend started and connected
+2022-01-26 12:15:54,401 INFO    MainThread:2226158 [wandb_init.py:init():533] updated telemetry
+2022-01-26 12:15:54,459 INFO    MainThread:2226158 [wandb_init.py:init():563] communicating current version
+2022-01-26 12:15:55,101 INFO    MainThread:2226158 [wandb_init.py:init():568] got version response 
+2022-01-26 12:15:55,101 INFO    MainThread:2226158 [wandb_init.py:init():578] communicating run to backend with 30 second timeout
+2022-01-26 12:15:55,277 INFO    MainThread:2226158 [wandb_init.py:init():606] starting run threads in backend
+2022-01-26 12:15:55,320 INFO    MainThread:2226158 [wandb_run.py:_console_start():1810] atexit reg
+2022-01-26 12:15:55,321 INFO    MainThread:2226158 [wandb_run.py:_redirect():1684] redirect: SettingsConsole.REDIRECT
+2022-01-26 12:15:55,322 INFO    MainThread:2226158 [wandb_run.py:_redirect():1689] Redirecting console.
+2022-01-26 12:15:55,325 INFO    MainThread:2226158 [wandb_run.py:_redirect():1745] Redirects installed.
+2022-01-26 12:15:55,325 INFO    MainThread:2226158 [wandb_init.py:init():633] run started, returning control to user process
+2022-01-26 12:15:55,341 INFO    MainThread:2226158 [wandb_run.py:_config_callback():956] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'architectures': ['Wav2Vec2ForPreTraining'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': 36, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'facebook/wav2vec2-xls-r-1b', 'transformers_version': '4.16.0.dev0', 'feat_extract_dropout': 0.0, 'model_type': 'wav2vec2', 'num_feat_extract_layers': 7, 'hidden_size': 1280, 'feat_extract_norm': 'layer', 'feat_extract_activation': 'gelu', 'conv_dim': [512, 512, 512, 512, 512, 512, 512], 'conv_stride': [5, 2, 2, 2, 2, 2, 2], 'conv_kernel': [10, 3, 3, 3, 3, 2, 2], 'conv_bias': True, 'num_conv_pos_embeddings': 128, 'num_conv_pos_embedding_groups': 16, 'num_hidden_layers': 48, 'intermediate_size': 5120, 'hidden_act': 'gelu', 'num_attention_heads': 16, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'activation_dropout': 0.0, 'feat_proj_dropout': 0.1, 'final_dropout': 0.0, 'layerdrop': 0.0, 'layer_norm_eps': 1e-05, 'initializer_range': 0.02, 'vocab_size': 39, 'do_stable_layer_norm': True, 'use_weighted_layer_sum': False, 'apply_spec_augment': True, 'mask_time_prob': 0.1, 'mask_time_length': 10, 'mask_time_min_masks': 2, 'mask_feature_prob': 0.0, 'mask_feature_length': 10, 'mask_feature_min_masks': 0, 'num_codevectors_per_group': 320, 'num_codevector_groups': 2, 'contrastive_logits_temperature': 0.1, 'feat_quantizer_dropout': 0.0, 'num_negatives': 100, 'codevector_dim': 1024, 'proj_codevector_dim': 1024, 'diversity_loss_weight': 0.1, 'ctc_loss_reduction': 'mean', 'ctc_zero_infinity': False, 'add_adapter': False, 'adapter_kernel_size': 3, 'adapter_stride': 2, 'num_adapter_layers': 3, 'output_hidden_size': 1280, 'classifier_proj_size': 256, 'tdnn_dim': [512, 512, 512, 512, 1500], 'tdnn_kernel': [5, 3, 3, 1, 1], 'tdnn_dilation': [1, 2, 3, 1, 1], 'xvector_output_dim': 512, 'output_dir': './1B', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 48, 'per_device_eval_batch_size': 48, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': 'None', 'learning_rate': 5e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 100.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.0, 'warmup_steps': 500, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './1B/runs/Jan26_12-15-06_ganymede.eafit.edu.co', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'amp', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': -1, 'xpu_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 250, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'cosine+drop_proj+low_specaugment-1b', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_hf', 'adafactor': False, 'group_by_length': True, 'length_column_name': 'input_length', 'report_to': "['wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': 'None', 'hub_model_id': 'None', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'gradient_checkpointing': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', '_n_gpu': 1, 'mp_parameters': '', 'train_batch_size': 48, 'eval_batch_size': 48}
+2022-01-26 12:15:55,344 INFO    MainThread:2226158 [wandb_watch.py:watch():43] Watching