add code for save checkpoint

main.py

@@ -9,6 +9,7 @@ from transformers.trainer_utils import get_last_checkpoint
 import json
 import os, glob
 from callbacks import BreakEachEpoch
+import subprocess
 
 logging.set_verbosity_info()
 
@@ -85,8 +86,14 @@ def load_prepared_dataset(path, processor, cache_file_filter_name, cache_file_ma
     return processed_dataset
 
 
-# def get_train_dataset():
-#     for i in range()
+def commit_checkpoint():
+    submit_commands = [
+        'git add model-bin/finetune/base/*',
+        'git commit -m "auto commit"',
+        'git push origin main'
+    ]
+    for command in submit_commands:
+        print(subprocess.run(command.split(), stdout=subprocess.PIPE).stdout.decode('utf-8'))
 
 if __name__ == "__main__":
 
@@ -208,3 +215,6 @@ if __name__ == "__main__":
         # Clear cache file to free disk
         test_dataset.cleanup_cache_files()
         train_dataset.cleanup_cache_files()
+
+        if epoch_idx % 10 == 0:
+          commit_checkpoint()