Async git push

app.py

@@ -6,6 +6,7 @@ import huggingface_hub
 import datetime
 import json
 import shutil
+import threading
 
 device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
 
@@ -34,6 +35,8 @@ repo = huggingface_hub.Repository(
 )
 repo.git_pull()
 
+lock = threading.Lock()
+
 class Interactive:
     def __init__(self):
         self.tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL_NAME, use_auth_token=HF_TOKEN_DOWNLOAD)
@@ -150,20 +153,15 @@ def predict(statements, do_saves):
         'False': 1 - output_raw['score_calibrated'],
     } for output_raw in output_raws]
     print('Logging statements to dataset:')
+    lock.acquire()
     for output_raw, do_save in zip(output_raws, do_saves):
         if do_save:
             print(output_raw)
             with open(DATA_PATH, 'a') as f:
                 json.dump(output_raw, f, ensure_ascii=False)
                 f.write('\n')
-    if any(do_saves):
-        try:
-            commit_url = repo.push_to_hub()
-        except Exception as e:
-            print('Failed to push to GitHub:', e)
-            repo.git_pull(rebase=True)
-        # print('Commit URL:', commit_url)
-    # print()
+    print()
+    lock.release()
     return outputs, output_raws, \
         [gr.update(visible=False) for _ in statements], \
         [gr.update(visible=True) for _ in statements], \
@@ -172,6 +170,7 @@ def predict(statements, do_saves):
 
 def record_feedback(output_raws, feedback, do_saves):
     print('Logging feedbacks to dataset:')
+    lock.acquire()
     for output_raw, do_save in zip(output_raws, do_saves):
         if do_save:
             output_raw.update({ 'feedback': feedback })
@@ -179,14 +178,8 @@ def record_feedback(output_raws, feedback, do_saves):
             with open(DATA_PATH, 'a') as f:
                 json.dump(output_raw, f, ensure_ascii=False)
                 f.write('\n')
-    if any(do_saves):
-        try:
-            commit_url = repo.push_to_hub()
-        except Exception as e:
-            print('Failed to push to GitHub:', e)
-            repo.git_pull(rebase=True)
-        # print('Commit URL:', commit_url)
-    # print()
+    print()
+    lock.release()
     return [gr.update(visible=True) for _ in output_raws], \
         [gr.update(visible=False) for _ in output_raws], \
         [gr.update(visible=False) for _ in output_raws], \
@@ -196,6 +189,18 @@ def record_feedback_agree(output_raws, do_saves):
 def record_feedback_disagree(output_raws, do_saves):
     return record_feedback(output_raws, 'disagree', do_saves)
 
+def push():
+    lock.acquire()
+    if repo.is_repo_clean():
+        print('No new data recorded, skipping git push ...')
+    else:
+        try:
+            commit_url = repo.push_to_hub()
+        except Exception as e:
+            print('Failed to push to git:', e)
+            repo.git_pull(rebase=True)
+    lock.release()
+
 examples = [
     # # openbookqa
     # 'If a person walks in the opposite direction of a compass arrow they are walking south.',
@@ -320,7 +325,6 @@ with gr.Blocks() as demo:
     feedback_agree.click(record_feedback_agree, inputs=[output_raw, do_save], outputs=[submit, feedback_agree, feedback_disagree, feedback_ack], batch=True, max_batch_size=16)
     feedback_disagree.click(record_feedback_disagree, inputs=[output_raw, do_save], outputs=[submit, feedback_agree, feedback_disagree, feedback_ack], batch=True, max_batch_size=16)
 
-demo.queue(concurrency_count=1).launch(debug=True)
+    demo.load(push, inputs=None, outputs=None, every=60) # Push to git every 60 seconds
 
-# Concurrency, Batching
-# Theme, CSS
+demo.queue(concurrency_count=1).launch(debug=True)