backup.py

from dotenv import load_dotenv
load_dotenv()
from datasets import load_dataset
from huggingface_hub import CommitScheduler
from huggingface_hub import HfApi
import os
import argparse

dataset_name = "Anuj-Panthri/JupyterLab_Storage"
work_dir = os.environ['NOTEBOOK_DIR']
dataset_save_path = "."

# def push_to_hub():
#     # backup "~/../../data/" to hugging face dataset "."
#     api = HfApi()

#     api.upload_folder(
#         repo_id=dataset_name,
#         repo_type="dataset",
#         folder_path=work_dir,
#         path_in_repo=dataset_save_path,        
#     )

# def commit_scheduler(minutes=10):

#     scheduler = CommitScheduler(
#         repo_id=dataset_name,
#         repo_type="dataset",
#         folder_path=work_dir,
#         path_in_repo=dataset_save_path,
#         every=minutes,
#     )

#     print("scheduler running")
#     while True:
#         pass


from git import Repo
import time

def push_to_hub():
    
    local_repo = Repo(work_dir)
    local_repo.git.add(all=True)
    print('Files Added Successfully') 
    local_repo.index.commit('Automatic backup commit')
    print('Commited successfully')
    origin = local_repo.remote(name='origin')
    origin.push()
    print('Pushed successfully')

def commit_scheduler(minutes=10):
    while(True):
        push_to_hub()
        time.sleep(minutes*60)


if __name__=="__main__":
    # Initialize parser
    parser = argparse.ArgumentParser()
    parser.add_argument("-p","--push", help="push to hub",action='store_true',default=True)
    parser.add_argument("-s","--schedule", help="push to hub scheduler every n minutes",type=int)
    args = parser.parse_args()

    if not args.schedule:
        push_to_hub()
    else:
        commit_scheduler(args.schedule)