commit

save_files.py

@@ -0,0 +1,22 @@
+import os
+from huggingface_hub import CommitScheduler
+
+hf_token = os.environ["HF_TOKEN"]
+
+# Dataset repo to push to
+REPO_ID = "sschellhammer/potato_annotation_data"
+
+# Folder to monitor
+LOCAL_FOLDER = "/app/output"
+
+# Commit interval (seconds)
+INTERVAL = 2  # every 2 minutes
+
+# Create CommitScheduler
+scheduler = CommitScheduler(
+    repo_id=REPO_ID,
+    token=HF_TOKEN,
+    folder_path=LOCAL_FOLDER,
+    repo_type="dataset",
+    every=INTERVAL
+)

start.sh

@@ -10,5 +10,7 @@ python setup_files.py
 echo "Listing files:"
 ls -l
 
+python save_files.py &
+
 echo "Starting Potato server..."
 exec potato start config.yaml -p 7860