add model

events.out.tfevents.1626217020.t1v-n-278acf21-w-0.60949.3.v2 → events.out.tfevents.1626420112.t1v-n-278acf21-w-0.561381.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f69d0cc1843d3c89a0326a1ed5576d47f5a88dcb34999a8a49d993eea40ef63
-size 3398110
+oid sha256:d8ee56f09af3471c76f8991f10060e5a16d7121ab1cf0cba9d7959393bb5c223
+size 220634

events.out.tfevents.1626448850.t1v-n-278acf21-w-0.590260.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fbe385b41508eae766e3ae9763a6bf8a20b0dad2a36c5058b526b6884a8433a
+size 662195

flax_model.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd33994b480ef0a93c7821a12df82c34656dc30539b623c1fb2050b1ba03be19
+size 190539834

src/run_persian.sh

@@ -19,7 +19,7 @@ export PER_DEVICE_EVAL_BATCH_SIZE=8
 #export GRADIENT_ACCUMULATION_STEPS=2
 export NUM_TRAIN_EPOCHS=5.0
 export LEARNING_RATE=5e-4
-export WARMUP_STEPS=1000
+export WARMUP_STEPS=2000
 export LOGGING_STEPS=500
 #export EVAL_STEPS=2500
 #export SAVE_STEPS=2500

src/run_wav2vec2_pretrain_flax.py

@@ -26,6 +26,7 @@ from typing import Dict, List, Optional, Union
 
 import numpy as np
 from datasets import DatasetDict, load_dataset
+from datasets import load_from_disk
 from tqdm import tqdm
 
 import flax
@@ -370,29 +371,33 @@ def main():
         return batch
 
     # load audio files into numpy arrays
-    vectorized_datasets = datasets.map(
-        prepare_dataset,
-        num_proc=data_args.preprocessing_num_workers,
-        remove_columns=datasets["train"].column_names
-    )
+    # vectorized_datasets = datasets.map(
+    #     prepare_dataset,
+    #     num_proc=data_args.preprocessing_num_workers,
+    #     remove_columns=datasets["train"].column_names
+    # )
 
     # filter audio files that are too long
-    vectorized_datasets = vectorized_datasets.filter(
-        lambda data: len(data["speech"]) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate)
-    )
+    # vectorized_datasets = vectorized_datasets.filter(
+    #     lambda data: len(data["speech"]) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate)
+    # )
 
-    def normalize(batch):
-        return feature_extractor(batch["speech"], sampling_rate=feature_extractor.sampling_rate)
+    # def normalize(batch):
+    #     return feature_extractor(batch["speech"], sampling_rate=feature_extractor.sampling_rate)
 
     # normalize and transform to `BatchFeatures`
-    vectorized_datasets = vectorized_datasets.map(
-        normalize,
-        batched=True,
-        num_proc=data_args.preprocessing_num_workers,
-        load_from_cache_file=not data_args.overwrite_cache,
-        remove_columns=vectorized_datasets["train"].column_names,
-    )
-    vectorized_datasets.save_to_disk(model_args.cache_dir)
+    # vectorized_datasets = vectorized_datasets.map(
+    #     normalize,
+    #     batched=True,
+    #     num_proc=data_args.preprocessing_num_workers,
+    #     load_from_cache_file=not data_args.overwrite_cache,
+    #     remove_columns=vectorized_datasets["train"].column_names,
+    # )
+    # vectorized_datasets.save_to_disk(model_args.cache_dir)
+
+    logger.info(f"Loading from {model_args.cache_dir}")
+    vectorized_datasets = load_from_disk(model_args.cache_dir)
+    logger.info(f"vectorized_datasets: {vectorized_datasets}")
 
     # pretraining is only supported for "newer" stable layer norm architecture
     # apply_spec_augment has to be True, mask_feature_prob has to be 0.0