upload

config.json

@@ -47,7 +47,7 @@
   "feat_proj_dropout": 0.0,
   "feat_quantizer_dropout": 0.0,
   "final_dropout": 0.0,
-  "gradient_checkpointing": true,
+  "gradient_checkpointing": false,
   "hidden_act": "gelu",
   "hidden_dropout": 0.0,
   "hidden_size": 1024,

run_pretrain_no_trainer.py

@@ -1,20 +1,20 @@
 #!/usr/bin/env python3
-import argparse
-import logging
-from dataclasses import dataclass
-from typing import Dict, List, Optional, Union
 import os
-
 import torch
 import math
 import datasets
+import wandb
+import argparse
+import logging
+import librosa
+import transformers
+
 from datasets import DatasetDict, load_dataset
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Union
 from accelerate import Accelerator, DeepSpeedPlugin
 from tqdm.auto import tqdm
 from torch.utils.data.dataloader import DataLoader
-
-import librosa
-import transformers
 from transformers import (
     MODEL_MAPPING,
     SchedulerType,
@@ -34,6 +34,9 @@ MODEL_CONFIG_CLASSES = list(MODEL_MAPPING.keys())
 MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
 
 
+wandb.init(project="pretraining-wav2vec2")
+
+
 def parse_args():
     parser = argparse.ArgumentParser(description="Finetune a transformers model on a text classification task")
     parser.add_argument(
@@ -505,6 +508,7 @@ def main():
                 for k, v in logs.items():
                     log_str += f"| {k}: {round(v.item(), 5)}"
 
+                wandb.log(logs)
                 progress_bar.write(log_str)
 
             if completed_steps >= args.max_train_steps: