GitPython

backup.py

@@ -10,30 +10,52 @@ dataset_name = "Anuj-Panthri/JupyterLab_Storage"
 work_dir = os.environ['NOTEBOOK_DIR']
 dataset_save_path = "."
 
-def push_to_hub():
-    # backup "~/../../data/" to hugging face dataset "."
-    api = HfApi()
+# def push_to_hub():
+#     # backup "~/../../data/" to hugging face dataset "."
+#     api = HfApi()
 
-    api.upload_folder(
-        repo_id=dataset_name,
-        repo_type="dataset",
-        folder_path=work_dir,
-        path_in_repo=dataset_save_path,        
-    )
+#     api.upload_folder(
+#         repo_id=dataset_name,
+#         repo_type="dataset",
+#         folder_path=work_dir,
+#         path_in_repo=dataset_save_path,        
+#     )
 
-def commit_scheduler(minutes=10):
+# def commit_scheduler(minutes=10):
+
+#     scheduler = CommitScheduler(
+#         repo_id=dataset_name,
+#         repo_type="dataset",
+#         folder_path=work_dir,
+#         path_in_repo=dataset_save_path,
+#         every=minutes,
+#     )
+
+#     print("scheduler running")
+#     while True:
+#         pass
+
+
+from git import Repo
+import time
+from glob import glob
 
-    scheduler = CommitScheduler(
-        repo_id=dataset_name,
-        repo_type="dataset",
-        folder_path=work_dir,
-        path_in_repo=dataset_save_path,
-        every=minutes,
-    )
 
-    print("scheduler running")
-    while True:
-        pass
+def push_to_hub():
+    local_repo = Repo(work_dir)
+    local_repo.index.add(glob(os.path.join(work_dir,"*")))
+    print('Files Added Successfully') 
+    local_repo.index.commit('Automatic backup commit')
+    print('Commited successfully')
+    origin = local_repo.remote(name='origin')
+    origin.push()
+    print('Pushed successfully')
+
+def commit_scheduler(minutes=10):
+    while(True):
+        push_to_hub()
+        time.sleep(minutes*60)
+
 
 if __name__=="__main__":
     # Initialize parser
@@ -46,6 +68,3 @@ if __name__=="__main__":
         push_to_hub()
     else:
         commit_scheduler(args.schedule)
-    
-    # print(args)
-    # commit_scheduler()

on_startup.sh

@@ -1,10 +1,5 @@
-#!/bin/bash
-# Write some commands here that will run on root user before startup.
-# For example, to clone transformers and install it in dev mode:
-# git clone https://github.com/huggingface/transformers.git
-# cd transformers && pip install -e ".[dev]"
-
-huggingface-cli login --token ${HF_TOKEN}
+git lfs install
+huggingface-cli login --token ${HF_TOKEN} --add-to-git-credential
 python3 restore.py
 nohup python3 backup.py -s 2 &
 exec ./start_server.sh

requirements.txt

@@ -4,4 +4,5 @@ tornado==6.2
 ipywidgets
 huggingface_hub
 datasets
-python-dotenv
+python-dotenv
+GitPython

restore.py

@@ -6,8 +6,13 @@ dataset_name = "Anuj-Panthri/JupyterLab_Storage"
 work_dir = os.environ['NOTEBOOK_DIR']
 dataset_save_path = "."
 
-snapshot_download(
-                    repo_id=dataset_name,
-                    repo_type="dataset",
-                    local_dir=work_dir,
-                )
+# snapshot_download(
+#                     repo_id=dataset_name,
+#                     repo_type="dataset",
+#                     local_dir=work_dir,
+#                 )
+
+
+import subprocess
+
+subprocess.run([f"git clone https://huggingface.co/datasets/Anuj-Panthri/JupyterLab_Storage {work_dir}"])