fix hf dataset storage

app.py

@@ -11,6 +11,12 @@ from src.utils import load_words, load_image_and_saliency, load_example_images
 from src.style import css
 from src.user import UserID
 
+from datetime import datetime
+from pathlib import Path
+from uuid import uuid4
+import json
+from huggingface_hub import CommitScheduler, HfApi
+
 def main():
     config = yaml.safe_load(open("config/config.yaml"))
     words = ['grad-cam', 'lime', 'sidu', 'rise']
@@ -131,18 +137,33 @@ def main():
 
         def save_results(answers):
             api = HfApi()
-            json_file_results = config['results']['exp1_dir']
+            json_file_results = config['results']['exp1_dir'] # 'exp1' 
             JSON_DATASET_DIR = Path("json_dataset")
             JSON_DATASET_DIR.mkdir(parents=True, exist_ok=True)
-            JSON_DATASET_PATH = JSON_DATASET_DIR / json_file_results
+            JSON_DATASET_PATH = JSON_DATASET_DIR / f"train-{uuid4()}.json"
+            scheduler = CommitScheduler(
+                repo_id="example-space-to-dataset-json",
+                repo_type="dataset",
+                folder_path=JSON_DATASET_DIR,
+                path_in_repo="data"
+            )
             
             info_to_push = {
                 "user_id": time.time(), 
-                "answer": {i: answer[i] for i in range(len(answer))}}
-
-            # use api to push the results to the hub 
-            api.push_to_hub(info_to_push, json_file_results, use_temp_dir=True)
-
+                "answer": {i: answer for i, answer in enumerate(answers)}
+            }
+
+            # Save the results into huggingface hub
+            with scheduler.lock:
+                with JSON_DATASET_PATH.open("a") as f:
+                    json.dump({
+                        "user_id": info_to_push["user_id"],
+                        "answers": info_to_push["answer"],
+                        "datetime": datetime.now().isoformat()
+                    }, f)
+                    f.write("\n")
+            scheduler.push_to_hub()
+            
  
         def add_answer(dropdown1,dropdown2,dropdown3,dropdown4, answers):
             rank = [dropdown1,dropdown2,dropdown3,dropdown4]