sync.py

import os, time, threading, shutil
from huggingface_hub import HfApi, snapshot_download

# NANOBOT路径对齐
NANOBOT_CONFIG = {"/root/.nanobot": "nanobot_settings", "/root/project/data": "data"}
# Hermes数据路径对齐
HERMES_CONFIG = {"/root/.hermes": "hermes_settings"}

IGNORE_LIST = ["*.db-shm", "*.db-wal", "models_dev_cache.json", "checkpoints/*", "*.pyc", "__pycache__", "hermes-agent/venv", "hermes-agent/.git", "logs/*", "cache/*", "*.pid", "*.lock","tmp/*","*.tmp", "__pycache__/*", "*.log.lock", "node_modules/*", "pkg/*", ".cache/*", ".npm/*","backup/*"]

DATASET_ID, HF_TOKEN = os.getenv("DATASET_ID"), os.getenv("HF_TOKEN")
api = HfApi()

def upload_all():
    if not DATASET_ID or not HF_TOKEN: return
    print(f"🚀 正在备份Nanobot数据...")
    for local_path, repo_path in NANOBOT_CONFIG.items():
        if os.path.exists(local_path) and os.listdir(local_path):
            try:
                api.upload_folder(folder_path=local_path, path_in_repo=repo_path,
                    repo_id=DATASET_ID, repo_type="dataset", token=HF_TOKEN, 
                    delete_patterns="*", ignore_patterns=IGNORE_LIST)
                print(f"✅ {repo_path} 备份成功")
            except Exception as e: print(f"❌ 备份翻车: {e}")
    
    # 备份Hermes数据
    print(f"🚀 正在备份Hermes数据...")
    for local_path, repo_path in HERMES_CONFIG.items():
        if os.path.exists(local_path):
            try:
                api.upload_folder(folder_path=local_path, path_in_repo=repo_path,
                    repo_id=DATASET_ID, repo_type="dataset", token=HF_TOKEN, 
                    delete_patterns="*", ignore_patterns=IGNORE_LIST)
                print(f"✅ {repo_path} 备份成功")
            except Exception as e: print(f"❌ Hermes备份翻车: {e}")

def download_all():
    if not DATASET_ID or not HF_TOKEN: return
    print("📥 正在恢复Nanobot数据...")
    for local_path, repo_path in SYNC_CONFIG.items():
        try:
            temp_dir = f"/tmp/hf_{repo_path}"
            os.makedirs(local_path, exist_ok=True)
            snapshot_download(repo_id=DATASET_ID, repo_type="dataset", local_dir=temp_dir, 
                              allow_patterns=f"{repo_path}/*", token=HF_TOKEN, local_dir_use_symlinks=False)
            source_dir = os.path.join(temp_dir, repo_path)
            if os.path.exists(source_dir):
                for item in os.listdir(source_dir):
                    s, d = os.path.join(source_dir, item), os.path.join(local_path, item)
                    if os.path.isdir(s):
                        if os.path.exists(d): shutil.rmtree(d)
                        shutil.copytree(s, d)
                    else: shutil.copy2(s, d)
            shutil.rmtree(temp_dir, ignore_errors=True)
            print(f"✅ {local_path} 恢复成功")
        except Exception as e: print(f"ℹ️ {repo_path} 恢复跳过")
    
    # 恢复Hermes数据
    print("📥 正在恢复Hermes数据...")
    for local_path, repo_path in HERMES_CONFIG.items():
        try:
            temp_dir = f"/tmp/hf_{repo_path}"
            os.makedirs(local_path, exist_ok=True)
            snapshot_download(repo_id=DATASET_ID, repo_type="dataset", local_dir=temp_dir, 
                              allow_patterns=f"{repo_path}/*", token=HF_TOKEN, local_dir_use_symlinks=False)
            source_dir = os.path.join(temp_dir, repo_path)
            if os.path.exists(source_dir):
                for item in os.listdir(source_dir):
                    s, d = os.path.join(source_dir, item), os.path.join(local_path, item)
                    if os.path.isdir(s):
                        if os.path.exists(d): shutil.rmtree(d)
                        shutil.copytree(s, d)
                    else: shutil.copy2(s, d)
            shutil.rmtree(temp_dir, ignore_errors=True)
            print(f"✅ {local_path} 恢复成功")
        except Exception as e: print(f"ℹ️ {repo_path} 恢复跳过")

if __name__ == "__main__":
    threading.Thread(target=lambda: [time.sleep(600) or upload_all() for _ in iter(int, 1)], daemon=True).start()
    while True: time.sleep(1)