Spaces:
Sleeping
Sleeping
File size: 1,751 Bytes
25ef662 9419101 a3f69ad 25ef662 9419101 25ef662 b2d3370 25ef662 b2d3370 25ef662 b2d3370 25ef662 2e4513f 96e4358 2e4513f 96e4358 2e4513f b2d3370 2e4513f b2d3370 25ef662 a3f69ad |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 |
from dotenv import load_dotenv
load_dotenv()
from datasets import load_dataset
from huggingface_hub import CommitScheduler
from huggingface_hub import HfApi
import os
import argparse
dataset_name = "Anuj-Panthri/JupyterLab_Storage"
work_dir = os.environ['NOTEBOOK_DIR']
dataset_save_path = "."
# def push_to_hub():
# # backup "~/../../data/" to hugging face dataset "."
# api = HfApi()
# api.upload_folder(
# repo_id=dataset_name,
# repo_type="dataset",
# folder_path=work_dir,
# path_in_repo=dataset_save_path,
# )
# def commit_scheduler(minutes=10):
# scheduler = CommitScheduler(
# repo_id=dataset_name,
# repo_type="dataset",
# folder_path=work_dir,
# path_in_repo=dataset_save_path,
# every=minutes,
# )
# print("scheduler running")
# while True:
# pass
from git import Repo
import time
def push_to_hub():
local_repo = Repo(work_dir)
local_repo.git.add(all=True)
print('Files Added Successfully')
local_repo.index.commit('Automatic backup commit')
print('Commited successfully')
origin = local_repo.remote(name='origin')
origin.push()
print('Pushed successfully')
def commit_scheduler(minutes=10):
while(True):
push_to_hub()
time.sleep(minutes*60)
if __name__=="__main__":
# Initialize parser
parser = argparse.ArgumentParser()
parser.add_argument("-p","--push", help="push to hub",action='store_true',default=True)
parser.add_argument("-s","--schedule", help="push to hub scheduler every n minutes",type=int)
args = parser.parse_args()
if not args.schedule:
push_to_hub()
else:
commit_scheduler(args.schedule)
|