sync.py

import os
import time
from huggingface_hub import HfApi, snapshot_download

# --- CONFIG ---
REPO_ID = "Sachin5112/Temp-storage" 
LOCAL_DIR = "/workspace/storage" # STICK TO THIS
TOKEN = os.environ.get("HF_TOKEN")

api = HfApi(token=TOKEN)

def pull():
    try:
        if not os.path.exists(LOCAL_DIR):
            os.makedirs(LOCAL_DIR)
        print("Pulling data from HF...")
        snapshot_download(repo_id=REPO_ID, repo_type="dataset", local_dir=LOCAL_DIR, token=TOKEN)
        print("Pull Success.")
    except Exception as e:
        print(f"Pull Error: {e}")

def push():
    try:
        print("Checking for changes...")
        api.upload_folder(
            folder_path=LOCAL_DIR, 
            repo_id=REPO_ID, 
            repo_type="dataset",
            delete_patterns="*", # This ensures deleted files on VNC are deleted on HF
        )
        print("Backup Success.")
    except Exception as e:
        print(f"Backup Error: {e}")

pull()

while True:
    time.sleep(60) # 1 minute is much safer than 1 second
    push()