File size: 1,437 Bytes
e2ce547 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | #!/usr/bin/env bash
set -euo pipefail
ENV_FILE="${ENV_FILE:-/etc/polymarket-collector.env}"
if [[ -f "$ENV_FILE" ]]; then
set -a
# shellcheck disable=SC1090
source "$ENV_FILE"
set +a
fi
DATA_DIR="${DATA_DIR:-/var/lib/polymarket-collector}"
RCLONE_REMOTE="${RCLONE_REMOTE:-}"
LOCAL_RETENTION_DAYS="${LOCAL_RETENTION_DAYS:-0}"
REMOVE_AFTER_UPLOAD="${REMOVE_AFTER_UPLOAD:-false}"
BACKUP_MIN_AGE="${BACKUP_MIN_AGE:-2m}"
if [[ -z "$RCLONE_REMOTE" ]]; then
echo "RCLONE_REMOTE is empty; backup is not configured."
exit 0
fi
if [[ ! -d "$DATA_DIR" ]]; then
echo "Data directory does not exist: $DATA_DIR" >&2
exit 1
fi
if ! [[ "$LOCAL_RETENTION_DAYS" =~ ^[0-9]+$ ]]; then
echo "LOCAL_RETENTION_DAYS must be a non-negative integer." >&2
exit 1
fi
case "${REMOVE_AFTER_UPLOAD,,}" in
1|true|yes) RCLONE_ACTION="move" ;;
0|false|no) RCLONE_ACTION="copy" ;;
*)
echo "REMOVE_AFTER_UPLOAD must be true or false." >&2
exit 1
;;
esac
LOCK_FILE="${BACKUP_LOCK_FILE:-/tmp/polymarket-backup.lock}"
exec 9>"$LOCK_FILE"
flock -n 9 || exit 0
rclone "$RCLONE_ACTION" "$DATA_DIR" "$RCLONE_REMOTE" \
--include "*.jsonl.gz" \
--min-age "$BACKUP_MIN_AGE" \
--checksum \
--transfers 2 \
--checkers 4 \
--retries 3 \
--low-level-retries 10 \
--create-empty-src-dirs
if (( LOCAL_RETENTION_DAYS > 0 )); then
find "$DATA_DIR" -type f -name "*.jsonl.gz" \
-mtime "+$LOCAL_RETENTION_DAYS" -delete
fi
|