add updated files

config.json

@@ -65,7 +65,7 @@
   "mask_time_length": 10,
   "mask_time_min_space": 1,
   "mask_time_other": 0.0,
-  "mask_time_prob": 0.18,
+  "mask_time_prob": 0.65,
   "mask_time_selection": "static",
   "model_type": "wav2vec2",
   "num_attention_heads": 16,

default_config.yaml

@@ -1,6 +1,6 @@
 compute_environment: LOCAL_MACHINE
 deepspeed_config:
-  gradient_accumulation_steps: 4 
+  gradient_accumulation_steps: 4
   offload_optimizer_device: cpu
   zero_stage: 2
 distributed_type: DEEPSPEED
@@ -10,4 +10,4 @@ main_process_ip: null
 main_process_port: null
 main_training_function: main
 num_machines: 0
-num_processes: 2
+num_processes: 8

run_main.sh

@@ -5,16 +5,18 @@ accelerate launch  --config_file ./default_config.yaml ./run_pretrain_no_trainer
 --max_train_steps="200000" \
 --num_warmup_steps="100000" \
 --gradient_accumulation_steps="4" \
---learning_rate="0.0001" \
+--learning_rate="0.005" \
 --weight_decay="0.01" \
---max_duration_in_seconds="8.0" \
+--max_duration_in_seconds="10.0" \
 --model_name_or_path="./" \
 --dataset_name="patrickvonplaten/librispeech_local" \
---manual_data_dir="/home/patrick/wav2vec2_reproduce" \
+--manual_data_dir="/home/ubuntu/wav2vec2_reproduce" \
 --dataset_config_name="clean" \
 --logging_steps="5" \
---per_device_train_batch_size="16" \
---per_device_eval_batch_size="16" \
+--per_device_train_batch_size="8" \
+--per_device_eval_batch_size="8" \
+#--per_device_train_batch_size="16" \
+#--per_device_eval_batch_size="16" \
 #--preprocessing_num_workers="4" \
 #--adam_beta1="0.9" \
 #--adam_beta2="0.98" \

run_pretrain_no_trainer.py

@@ -34,9 +34,6 @@ MODEL_CONFIG_CLASSES = list(MODEL_MAPPING.keys())
 MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
 
 
-wandb.init(project="pretraining-wav2vec2")
-
-
 def parse_args():
     parser = argparse.ArgumentParser(description="Finetune a transformers model on a text classification task")
     parser.add_argument(
@@ -330,6 +327,8 @@ def main():
     if accelerator.is_local_main_process:
         datasets.utils.logging.set_verbosity_warning()
         transformers.utils.logging.set_verbosity_info()
+
+        wandb.init(project="pretraining-wav2vec2")
     else:
         datasets.utils.logging.set_verbosity_error()
         transformers.utils.logging.set_verbosity_error()
@@ -381,9 +380,6 @@ def main():
             split="train",
         )
 
-#    raw_datasets["train"] = raw_datasets["train"].select(range(128))
-#    raw_datasets["validation"] = raw_datasets["validation"].select(range(16))
-
     # only normalized-inputs-training is supported
     feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
         args.model_name_or_path, do_normalize=True
@@ -489,9 +485,9 @@ def main():
                 gumbel_temperature = max(args.max_gumbel_temperature * args.gumbel_temperature_decay ** completed_steps, args.min_gumbel_temperature)
 
                 if hasattr(model, "module"):
-                    model = model.module
-
-                model.set_gumbel_temperature(gumbel_temperature)
+                    model.module.set_gumbel_temperature(gumbel_temperature)
+                else:
+                    model.set_gumbel_temperature(gumbel_temperature)
 
             if step % args.logging_steps == 0:
                 logs = {
@@ -508,8 +504,9 @@ def main():
                 for k, v in logs.items():
                     log_str += f"| {k}: {round(v.item(), 5)}"
 
-                wandb.log(logs)
-                progress_bar.write(log_str)
+                if accelerator.is_local_main_process:
+                    wandb.log(logs)
+                    progress_bar.write(log_str)
 
             if completed_steps >= args.max_train_steps:
                 break