Upload sync_data.py

sync_data.py

@@ -0,0 +1,83 @@
+import os
+import shutil
+import time
+import threading
+from pathlib import Path
+from datetime import datetime
+from huggingface_hub import HfApi, hf_hub_download, list_repo_files
+
+DATA_DIR = Path(os.environ.get("DATA_DIR", "/root/.openclaw"))
+DATASET_REPO_ID = os.environ.get("DATASET_REPO_ID", "your-username/your-dataset")
+SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", 300))
+ARCHIVE_NAME = "openclaw_backup.tar.gz"
+
+api = HfApi()
+
+
+def download_from_dataset():
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+    try:
+        files = list_repo_files(repo_id=DATASET_REPO_ID, repo_type="dataset")
+        if ARCHIVE_NAME in files:
+            print(f"[{datetime.now()}] Downloading {ARCHIVE_NAME}...")
+            archive_path = hf_hub_download(
+                repo_id=DATASET_REPO_ID,
+                filename=ARCHIVE_NAME,
+                repo_type="dataset",
+                local_dir="/tmp"
+            )
+            shutil.unpack_archive(archive_path, DATA_DIR)
+            os.remove(archive_path)
+            print(f"[{datetime.now()}] Data restored to {DATA_DIR}")
+        else:
+            print(f"[{datetime.now()}] No backup found, starting fresh")
+    except Exception as e:
+        print(f"[{datetime.now()}] Download failed: {e}")
+
+
+def upload_to_dataset():
+    if not any(DATA_DIR.iterdir()):
+        print(f"[{datetime.now()}] No files to upload")
+        return
+    try:
+        archive_path = Path("/tmp") / ARCHIVE_NAME
+        shutil.make_archive(
+            archive_path.with_suffix("").with_suffix(""),
+            "gztar",
+            DATA_DIR
+        )
+        print(f"[{datetime.now()}] Uploading {ARCHIVE_NAME}...")
+        api.upload_file(
+            path_or_fileobj=str(archive_path),
+            path_in_repo=ARCHIVE_NAME,
+            repo_id=DATASET_REPO_ID,
+            repo_type="dataset",
+        )
+        os.remove(archive_path)
+        print(f"[{datetime.now()}] Upload completed")
+    except Exception as e:
+        print(f"[{datetime.now()}] Upload failed: {e}")
+
+
+def sync_loop():
+    while True:
+        time.sleep(SYNC_INTERVAL)
+        upload_to_dataset()
+
+
+def main():
+    print(f"Dataset: {DATASET_REPO_ID}")
+    print(f"Data dir: {DATA_DIR}")
+    print(f"Sync interval: {SYNC_INTERVAL}s")
+
+    download_from_dataset()
+
+    sync_thread = threading.Thread(target=sync_loop, daemon=True)
+    sync_thread.start()
+
+    while True:
+        time.sleep(3600)
+
+
+if __name__ == "__main__":
+    main()