Training in progress, step 210

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6cb075847c29394668f005f6008d8c158e9c910f1e463c7995b684773086b51b
+oid sha256:bcee542494f06b4b7902a83fcfbb72ff945979831bd1faced0e483338bde34c8
 size 151098921

runs/Dec13_14-14-43_d7f040c448a8/1670940896.9428296/events.out.tfevents.1670940896.d7f040c448a8.61677.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f62ad83e8a4c195d87e54b52bcb8f0cd2c676df4575191e4f714f4b2cf504b41
+size 5883

runs/Dec13_14-14-43_d7f040c448a8/events.out.tfevents.1670940896.d7f040c448a8.61677.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac2bde0e56a2cfc958bf1d1ed3f685a88c6b89fbea8bbba5dd6e5aaf86d58b2d
+size 4745

src/run_debug.sh

@@ -7,7 +7,7 @@ python src/run_speech_recognition_seq2seq_streaming.py \
 	--eval_split_name="validation" \
 	--model_index_name="Whisper Tiny Belarusian" \
 	\
-	--max_steps="200" \
+	--max_steps="300" \
 	--max_eval_samples="64" \
 	--output_dir="./" \
 	--per_device_train_batch_size="32" \
@@ -35,7 +35,6 @@ python src/run_speech_recognition_seq2seq_streaming.py \
 	\
 	--do_train \
 	--do_eval \
-	--overwrite_output_dir \
 	--ignore_data_skip \
 	--predict_with_generate \
 	--do_normalize_eval \

src/run_speech_recognition_seq2seq_streaming.py

@@ -23,6 +23,7 @@ with 🤗 Datasets' streaming mode.
 import logging
 import os
 import sys
+import datetime
 import re
 import regex
 import unicodedata
@@ -336,10 +337,14 @@ def main():
 
 
     # 2. Setup logging
+    now_str = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
     logging.basicConfig(
         format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
         datefmt="%m/%d/%Y %H:%M:%S",
-        handlers=[logging.StreamHandler(sys.stdout)],
+        handlers=[
+            logging.StreamHandler(sys.stdout),
+            logging.FileHandler(filename=f'train_{now_str}.log', mode='w')
+        ],
     )
     log_level = training_args.get_process_log_level()
     logger.setLevel(log_level)

train.log

@@ -223,3 +223,4 @@
   eval_samples_per_second =      3.888
   eval_steps_per_second   =      0.122
   eval_wer                =    51.2821
+{'loss': 0.3956, 'learning_rate': 3.206896551724138e-05, 'epoch': 0.03}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7060ae76bbcda709483e8d25020c5fe10166a869e6cd092088018350b53a30f
+oid sha256:350c7e60ed35dd7fe825968ffb77600485c958e44f8b775cc30447b1f2b880aa
 size 3643