File size: 2,333 Bytes
52c0a32 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | #!/usr/bin/env bash
# Sync local data into a Hugging Face bucket.
# Defaults are loaded from backend/.env when present.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"
if [ -f ".env" ]; then
while IFS= read -r raw_line || [ -n "$raw_line" ]; do
line="${raw_line%$'\r'}"
case "$line" in
''|'#'*) continue ;;
esac
if [[ "$line" == *=* ]]; then
key="${line%%=*}"
value="${line#*=}"
key="$(echo "$key" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
if [[ ! "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
continue
fi
if [[ "$value" =~ ^\".*\"$ ]]; then
value="${value:1:${#value}-2}"
elif [[ "$value" =~ ^\'.*\'$ ]]; then
value="${value:1:${#value}-2}"
else
value="$(echo "$value" | sed 's/[[:space:]]#.*$//;s/[[:space:]]*$//')"
fi
export "$key=$value"
fi
done < ./.env
fi
BUCKET_URI="${HF_BUCKET_URI:-hf://buckets/<username>/<bucket-name>}"
LOCAL_DIR="${1:-${HF_BUCKET_LOCAL_DIR:-./data}}"
DELETE_FLAG="${HF_BUCKET_DELETE:-false}"
# Force host-safe HF cache path for CLI operations.
HF_HOME_HOST_DEFAULT="${HOME:-$PWD}/.cache/huggingface"
HF_HOME="${HF_HOME_HOST:-${DEPLOY_HF_HOME:-$HF_HOME_HOST_DEFAULT}}"
export HF_HOME
if [[ "$BUCKET_URI" == *"<username>"* ]] || [[ "$BUCKET_URI" == *"<bucket-name>"* ]]; then
echo "ERROR: HF_BUCKET_URI is still a placeholder: $BUCKET_URI"
echo "Set HF_BUCKET_URI in backend/.env to your real bucket URI."
exit 1
fi
if ! command -v hf >/dev/null 2>&1; then
echo "ERROR: Hugging Face CLI (hf) is not installed."
echo "Install guide: https://hf.co/docs/huggingface_hub/guides/cli"
exit 1
fi
if ! hf auth whoami >/dev/null 2>&1; then
echo "ERROR: Hugging Face CLI is not authenticated. Run: hf auth login"
exit 1
fi
if [ ! -d "$LOCAL_DIR" ]; then
echo "ERROR: Local directory does not exist: $LOCAL_DIR"
exit 1
fi
echo "Syncing local directory to HF bucket"
echo " Local : $LOCAL_DIR"
echo " Bucket: $BUCKET_URI"
if [ "$DELETE_FLAG" = "true" ]; then
echo " Mode : mirror (delete remote files not present locally)"
hf sync "$LOCAL_DIR" "$BUCKET_URI" --delete
else
echo " Mode : additive (no remote deletes)"
hf sync "$LOCAL_DIR" "$BUCKET_URI"
fi
echo "Bucket sync completed successfully."
|