File size: 4,142 Bytes
cd76221 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | import os, time, threading, shutil
from huggingface_hub import HfApi, snapshot_download
# NANOBOT路径对齐
NANOBOT_CONFIG = {"/root/.nanobot": "nanobot_settings", "/root/project/data": "data"}
# Hermes数据路径对齐
HERMES_CONFIG = {"/root/.hermes": "hermes_settings"}
IGNORE_LIST = ["*.db-shm", "*.db-wal", "models_dev_cache.json", "checkpoints/*", "*.pyc", "__pycache__", "hermes-agent/venv", "hermes-agent/.git", "logs/*", "cache/*", "*.pid", "*.lock","tmp/*","*.tmp", "__pycache__/*", "*.log.lock", "node_modules/*", "pkg/*", ".cache/*", ".npm/*","backup/*"]
DATASET_ID, HF_TOKEN = os.getenv("DATASET_ID"), os.getenv("HF_TOKEN")
api = HfApi()
def upload_all():
if not DATASET_ID or not HF_TOKEN: return
print(f"🚀 正在备份Nanobot数据...")
for local_path, repo_path in NANOBOT_CONFIG.items():
if os.path.exists(local_path) and os.listdir(local_path):
try:
api.upload_folder(folder_path=local_path, path_in_repo=repo_path,
repo_id=DATASET_ID, repo_type="dataset", token=HF_TOKEN,
delete_patterns="*", ignore_patterns=IGNORE_LIST)
print(f"✅ {repo_path} 备份成功")
except Exception as e: print(f"❌ 备份翻车: {e}")
# 备份Hermes数据
print(f"🚀 正在备份Hermes数据...")
for local_path, repo_path in HERMES_CONFIG.items():
if os.path.exists(local_path):
try:
api.upload_folder(folder_path=local_path, path_in_repo=repo_path,
repo_id=DATASET_ID, repo_type="dataset", token=HF_TOKEN,
delete_patterns="*", ignore_patterns=IGNORE_LIST)
print(f"✅ {repo_path} 备份成功")
except Exception as e: print(f"❌ Hermes备份翻车: {e}")
def download_all():
if not DATASET_ID or not HF_TOKEN: return
print("📥 正在恢复Nanobot数据...")
for local_path, repo_path in SYNC_CONFIG.items():
try:
temp_dir = f"/tmp/hf_{repo_path}"
os.makedirs(local_path, exist_ok=True)
snapshot_download(repo_id=DATASET_ID, repo_type="dataset", local_dir=temp_dir,
allow_patterns=f"{repo_path}/*", token=HF_TOKEN, local_dir_use_symlinks=False)
source_dir = os.path.join(temp_dir, repo_path)
if os.path.exists(source_dir):
for item in os.listdir(source_dir):
s, d = os.path.join(source_dir, item), os.path.join(local_path, item)
if os.path.isdir(s):
if os.path.exists(d): shutil.rmtree(d)
shutil.copytree(s, d)
else: shutil.copy2(s, d)
shutil.rmtree(temp_dir, ignore_errors=True)
print(f"✅ {local_path} 恢复成功")
except Exception as e: print(f"ℹ️ {repo_path} 恢复跳过")
# 恢复Hermes数据
print("📥 正在恢复Hermes数据...")
for local_path, repo_path in HERMES_CONFIG.items():
try:
temp_dir = f"/tmp/hf_{repo_path}"
os.makedirs(local_path, exist_ok=True)
snapshot_download(repo_id=DATASET_ID, repo_type="dataset", local_dir=temp_dir,
allow_patterns=f"{repo_path}/*", token=HF_TOKEN, local_dir_use_symlinks=False)
source_dir = os.path.join(temp_dir, repo_path)
if os.path.exists(source_dir):
for item in os.listdir(source_dir):
s, d = os.path.join(source_dir, item), os.path.join(local_path, item)
if os.path.isdir(s):
if os.path.exists(d): shutil.rmtree(d)
shutil.copytree(s, d)
else: shutil.copy2(s, d)
shutil.rmtree(temp_dir, ignore_errors=True)
print(f"✅ {local_path} 恢复成功")
except Exception as e: print(f"ℹ️ {repo_path} 恢复跳过")
if __name__ == "__main__":
threading.Thread(target=lambda: [time.sleep(600) or upload_all() for _ in iter(int, 1)], daemon=True).start()
while True: time.sleep(1) |