#!/bin/bash REPO_URL="https://huggingface.co/datasets/Shuyun18/hermes" LOCAL_REPO="/tmp/hermes_data" HERMES_DIR="/root/.hermes" HERMES_HUDUI_DIR="/root/.hermes-hudui" # 防止 Git 因交互式提示挂起 export GIT_TERMINAL_PROMPT=0 export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no" if [ -z "$HF_TOKEN" ]; then echo "Warning: HF_TOKEN not found. Data sync disabled." exit 0 fi # 配置 Git 用户信息 git config --global user.name "Shuyun18" git config --global user.email "shuyun18@users.noreply.huggingface.co" # 验证当前认证状态 pip install huggingface_hub -q echo "=== HF Authentication Status ===" hf whoami echo "================================" # 使用带 token 的 URL 进行 git 操作 REPO_URL_WITH_TOKEN="https://Shuyun18:${HF_TOKEN}@huggingface.co/datasets/Shuyun18/hermes" if [ "$1" == "load" ]; then echo "Loading data from dataset..." rm -rf "$LOCAL_REPO" git clone "$REPO_URL_WITH_TOKEN" "$LOCAL_REPO" || echo "Dataset repo is empty or not found." # 确保本地核心目录存在 mkdir -p "$HERMES_DIR" "$HERMES_HUDUI_DIR" # 确保从仓库拉取的子目录存在,避免 rsync 报错 mkdir -p "$LOCAL_REPO/hermes" "$LOCAL_REPO/hermes-hudui" # 将子目录数据分别同步到对应位置 rsync -av --exclude='.git' "$LOCAL_REPO/hermes/" "$HERMES_DIR/" rsync -av --exclude='.git' "$LOCAL_REPO/hermes-hudui/" "$HERMES_HUDUI_DIR/" echo "Data loaded." elif [ "$1" == "save" ]; then echo "Saving data to dataset..." # 先 clone 仓库(确保有正确的 git 历史) rm -rf "$LOCAL_REPO" git clone "$REPO_URL_WITH_TOKEN" "$LOCAL_REPO" || { echo "Failed to clone repo, creating new one..." mkdir -p "$LOCAL_REPO" cd "$LOCAL_REPO" git init git checkout -b main git remote add origin "$REPO_URL_WITH_TOKEN" } cd "$LOCAL_REPO" # 确保本地和目标子目录存在 mkdir -p "$LOCAL_REPO/hermes" "$LOCAL_REPO/hermes-hudui" mkdir -p "$HERMES_DIR" "$HERMES_HUDUI_DIR" rsync -av --delete --exclude='.git' "$HERMES_DIR/" "$LOCAL_REPO/hermes/" rsync -av --delete --exclude='.git' "$HERMES_HUDUI_DIR/" "$LOCAL_REPO/hermes-hudui/" git add . git commit -m "Auto-sync data at $(date)" || echo "Nothing to commit" git push -u origin main || echo "Push failed" echo "Data saved." fi