Update hf-sync.sh
Browse files- hf-sync.sh +23 -36
hf-sync.sh
CHANGED
|
@@ -5,9 +5,7 @@ HF_DATASET_ID="${HF_DATASET_ID:-}"
|
|
| 5 |
LOCAL_DIR="${OPENCLAW_HOME:-/home/user}/.openclaw"
|
| 6 |
CLONE_DIR="/tmp/hf-dataset-sync"
|
| 7 |
|
| 8 |
-
log() {
|
| 9 |
-
echo "[hf-sync] $1"
|
| 10 |
-
}
|
| 11 |
|
| 12 |
if [ -z "$HF_TOKEN" ] || [ -z "$HF_DATASET_ID" ]; then
|
| 13 |
log "HF_TOKEN or HF_DATASET_ID not set, skipping"
|
|
@@ -17,26 +15,24 @@ fi
|
|
| 17 |
REPO_URL="https://user:${HF_TOKEN}@huggingface.co/datasets/${HF_DATASET_ID}"
|
| 18 |
|
| 19 |
git config --global user.email "openclaw@hf.space" 2>/dev/null || true
|
| 20 |
-
git config --global user.name "OpenClaw Sync"
|
| 21 |
-
git config --global http.postBuffer 52428800
|
| 22 |
|
| 23 |
-
|
| 24 |
rm -rf "$CLONE_DIR"
|
| 25 |
git clone --depth 1 --quiet "$REPO_URL" "$CLONE_DIR" 2>/dev/null && return 0
|
| 26 |
mkdir -p "$CLONE_DIR"
|
| 27 |
cd "$CLONE_DIR"
|
| 28 |
git init --quiet
|
| 29 |
git remote add origin "$REPO_URL"
|
| 30 |
-
return 0
|
| 31 |
}
|
| 32 |
|
| 33 |
restore() {
|
| 34 |
-
log "Restoring from ${HF_DATASET_ID}
|
| 35 |
if [ -d "$CLONE_DIR/.git" ]; then
|
| 36 |
-
cd "$CLONE_DIR"
|
| 37 |
-
git pull --quiet 2>/dev/null || true
|
| 38 |
else
|
| 39 |
-
|
| 40 |
fi
|
| 41 |
if [ -d "$CLONE_DIR/openclaw" ]; then
|
| 42 |
mkdir -p "$LOCAL_DIR"
|
|
@@ -48,20 +44,18 @@ fi
|
|
| 48 |
}
|
| 49 |
|
| 50 |
backup() {
|
| 51 |
-
log "Backing up to ${HF_DATASET_ID}
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
return 0
|
| 55 |
-
fi
|
| 56 |
if [ -d "$CLONE_DIR/.git" ]; then
|
| 57 |
-
cd "$CLONE_DIR"
|
| 58 |
-
git pull --quiet 2>/dev/null || true
|
| 59 |
else
|
| 60 |
-
|
| 61 |
fi
|
|
|
|
| 62 |
cd "$CLONE_DIR"
|
| 63 |
-
rm -rf openclaw
|
| 64 |
-
|
| 65 |
find "$LOCAL_DIR" -maxdepth 1 -mindepth 1 | while read item; do
|
| 66 |
name=$(basename "$item")
|
| 67 |
case "$name" in
|
|
@@ -69,17 +63,13 @@ case "$name" in
|
|
| 69 |
*) cp -r "$item" "openclaw/$name" ;;
|
| 70 |
esac
|
| 71 |
done
|
|
|
|
| 72 |
git add -A
|
| 73 |
-
|
| 74 |
-
log "No changes, skip push"
|
| 75 |
-
return 0
|
| 76 |
-
fi
|
| 77 |
git commit -m "sync $(date -u +%Y-%m-%dT%H:%M:%SZ)" --quiet
|
| 78 |
-
git push --quiet "$REPO_URL" HEAD:main 2>/dev/null ||
|
| 79 |
-
git push --quiet "$REPO_URL" HEAD:master 2>/dev/null ||
|
| 80 |
-
log "Push failed, retry next cycle"
|
| 81 |
-
return 0
|
| 82 |
-
}
|
| 83 |
log "Backup done"
|
| 84 |
}
|
| 85 |
|
|
@@ -87,15 +77,12 @@ background_loop() {
|
|
| 87 |
INTERVAL="${HF_SYNC_INTERVAL:-3600}"
|
| 88 |
sleep 300
|
| 89 |
backup
|
| 90 |
-
while true; do
|
| 91 |
-
sleep "$INTERVAL"
|
| 92 |
-
backup
|
| 93 |
-
done
|
| 94 |
}
|
| 95 |
|
| 96 |
case "${1:-restore}" in
|
| 97 |
restore) restore ;;
|
| 98 |
-
backup) backup
|
| 99 |
loop) background_loop ;;
|
| 100 |
*) log "Usage: hf-sync.sh [restore|backup|loop]"; exit 1 ;;
|
| 101 |
-
esac
|
|
|
|
| 5 |
LOCAL_DIR="${OPENCLAW_HOME:-/home/user}/.openclaw"
|
| 6 |
CLONE_DIR="/tmp/hf-dataset-sync"
|
| 7 |
|
| 8 |
+
log() { echo "[hf-sync] $1"; }
|
|
|
|
|
|
|
| 9 |
|
| 10 |
if [ -z "$HF_TOKEN" ] || [ -z "$HF_DATASET_ID" ]; then
|
| 11 |
log "HF_TOKEN or HF_DATASET_ID not set, skipping"
|
|
|
|
| 15 |
REPO_URL="https://user:${HF_TOKEN}@huggingface.co/datasets/${HF_DATASET_ID}"
|
| 16 |
|
| 17 |
git config --global user.email "openclaw@hf.space" 2>/dev/null || true
|
| 18 |
+
git config --global user.name "OpenClaw Sync" 2>/dev/null || true
|
| 19 |
+
git config --global http.postBuffer 52428800 2>/dev/null || true
|
| 20 |
|
| 21 |
+
clone_or_init() {
|
| 22 |
rm -rf "$CLONE_DIR"
|
| 23 |
git clone --depth 1 --quiet "$REPO_URL" "$CLONE_DIR" 2>/dev/null && return 0
|
| 24 |
mkdir -p "$CLONE_DIR"
|
| 25 |
cd "$CLONE_DIR"
|
| 26 |
git init --quiet
|
| 27 |
git remote add origin "$REPO_URL"
|
|
|
|
| 28 |
}
|
| 29 |
|
| 30 |
restore() {
|
| 31 |
+
log "Restoring from ${HF_DATASET_ID}..."
|
| 32 |
if [ -d "$CLONE_DIR/.git" ]; then
|
| 33 |
+
cd "$CLONE_DIR" && git pull --quiet 2>/dev/null || true
|
|
|
|
| 34 |
else
|
| 35 |
+
clone_or_init || { log "Clone failed, starting fresh"; return 0; }
|
| 36 |
fi
|
| 37 |
if [ -d "$CLONE_DIR/openclaw" ]; then
|
| 38 |
mkdir -p "$LOCAL_DIR"
|
|
|
|
| 44 |
}
|
| 45 |
|
| 46 |
backup() {
|
| 47 |
+
log "Backing up to ${HF_DATASET_ID}..."
|
| 48 |
+
[ -d "$LOCAL_DIR" ] || { log "Nothing to backup"; return 0; }
|
| 49 |
+
|
|
|
|
|
|
|
| 50 |
if [ -d "$CLONE_DIR/.git" ]; then
|
| 51 |
+
cd "$CLONE_DIR" && git pull --quiet 2>/dev/null || true
|
|
|
|
| 52 |
else
|
| 53 |
+
clone_or_init || { log "Clone failed, skip backup"; return 0; }
|
| 54 |
fi
|
| 55 |
+
|
| 56 |
cd "$CLONE_DIR"
|
| 57 |
+
rm -rf openclaw && mkdir -p openclaw
|
| 58 |
+
|
| 59 |
find "$LOCAL_DIR" -maxdepth 1 -mindepth 1 | while read item; do
|
| 60 |
name=$(basename "$item")
|
| 61 |
case "$name" in
|
|
|
|
| 63 |
*) cp -r "$item" "openclaw/$name" ;;
|
| 64 |
esac
|
| 65 |
done
|
| 66 |
+
|
| 67 |
git add -A
|
| 68 |
+
git diff --cached --quiet && { log "No changes"; return 0; }
|
|
|
|
|
|
|
|
|
|
| 69 |
git commit -m "sync $(date -u +%Y-%m-%dT%H:%M:%SZ)" --quiet
|
| 70 |
+
git push --quiet "$REPO_URL" HEAD:main 2>/dev/null ||
|
| 71 |
+
git push --quiet "$REPO_URL" HEAD:master 2>/dev/null ||
|
| 72 |
+
{ log "Push failed, retry next cycle"; return 0; }
|
|
|
|
|
|
|
| 73 |
log "Backup done"
|
| 74 |
}
|
| 75 |
|
|
|
|
| 77 |
INTERVAL="${HF_SYNC_INTERVAL:-3600}"
|
| 78 |
sleep 300
|
| 79 |
backup
|
| 80 |
+
while true; do sleep "$INTERVAL" && backup; done
|
|
|
|
|
|
|
|
|
|
| 81 |
}
|
| 82 |
|
| 83 |
case "${1:-restore}" in
|
| 84 |
restore) restore ;;
|
| 85 |
+
backup) backup ;;
|
| 86 |
loop) background_loop ;;
|
| 87 |
*) log "Usage: hf-sync.sh [restore|backup|loop]"; exit 1 ;;
|
| 88 |
+
esac
|