#!/bin/bash # --- Helper Functions --- # Function to check if Cloudreve is running by checking the port wait_for_cloudreve() { echo "[Backup Manager] Waiting for Cloudreve service (port 5212)..." while ! nc -z 127.0.0.1 5212; do echo "[Backup Manager] Cloudreve not ready yet, sleeping 5s..." sleep 5 done echo "[Backup Manager] Cloudreve service detected." } # Function to perform a single backup action perform_backup_once() { echo "[Backup Action] Performing single backup..." local config_path="$1" local db_path="$2" local exe_path="$3" local cloudreve_dir="$4" local backup_prefix="$5" # Wait brief moment to ensure files are stable after service start sleep 5 if [ ! -f "$config_path" ] || [ ! -f "$db_path" ] || [ ! -f "$exe_path" ]; then echo "[Backup Action] WARN: Essential files missing for backup ($config_path, $db_path, $exe_path). Skipping." return 1 fi local timestamp=$(date +%Y%m%d_%H%M%S) local backup_file="${backup_prefix}_${timestamp}.tar.gz" local backup_path="/tmp/${backup_file}" echo "[Backup Action] Compressing data (exec, db, config) to: $backup_path" tar -czf "$backup_path" -C "$cloudreve_dir" \ "$(basename "$exe_path")" \ "$(basename "$db_path")" \ "$(basename "$config_path")" \ || { echo "[Backup Action] ERROR: tar command failed."; rm -f "$backup_path"; return 1; } # Add error check if [ -s "$backup_path" ]; then echo "[Backup Action] Compression complete. Size: $(ls -lh "$backup_path" | awk '{print $5}')" echo "[Backup Action] Uploading backup to HuggingFace..." # Call the Python upload_backup function upload_backup "$backup_path" "${backup_file}" local upload_status=$? if [ $upload_status -ne 0 ]; then echo "[Backup Action] ERROR: Backup upload failed. Keeping local archive: $backup_path" return 1 else echo "[Backup Action] Upload successful. Removing local archive." rm -f "$backup_path" return 0 fi else echo "[Backup Action] ERROR: Compression failed or created empty file. Skipping upload." rm -f "$backup_path" return 1 fi } # --- Python Function Definitions (Ensure these are complete and correct) --- # Python Function: Upload Backup upload_backup() { local file_path="$1"; local file_name="$2"; local token="$HF_TOKEN"; local repo_id="$DATASET_ID" echo "[Python Upload] Preparing: $file_path as $file_name to $repo_id" python3 -c " import sys, os, time from huggingface_hub import HfApi, list_repo_files, delete_file, upload_file print(f'[Python Upload] HF_TOKEN is set: {os.environ.get(\"HF_TOKEN\") is not None}') print(f'[Python Upload] DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}') def manage_backups(api, repo_id_val, backup_prefix_val, max_files=50): # Increased max_files print('[Python Upload] Managing old backups...') try: # Retry logic for listing files, as sometimes HF API might be slow/flaky for attempt in range(3): try: files = list_repo_files(repo_id=repo_id_val, repo_type='dataset', token=os.environ.get('HF_TOKEN')) break # Success except Exception as list_err: print(f'[Python Upload] Error listing files (attempt {attempt+1}/3): {list_err}') if attempt == 2: raise # Raise after last attempt time.sleep(5) # Wait before retrying backup_files = sorted([f for f in files if f.startswith(backup_prefix_val) and f.endswith('.tar.gz')]) print(f'[Python Upload] Found {len(backup_files)} existing backup files.') if len(backup_files) >= max_files: num_to_delete = len(backup_files) - max_files + 1 print(f'[Python Upload] Max backups ({max_files}) reached. Need to delete {num_to_delete} oldest backups.') files_to_delete = backup_files[:num_to_delete] for file_to_delete in files_to_delete: try: print(f'[Python Upload] Deleting old backup: {file_to_delete}') delete_file(path_in_repo=file_to_delete, repo_id=repo_id_val, repo_type='dataset', token=os.environ.get('HF_TOKEN')) print(f'[Python Upload] Successfully deleted: {file_to_delete}') except Exception as e: print(f'[Python Upload] ERROR deleting {file_to_delete}: {str(e)}') # Log error but continue else: print(f'[Python Upload] Number of backups ({len(backup_files)}) is within the limit ({max_files}).') except Exception as e: print(f'[Python Upload] ERROR during backup management: {e}') api = HfApi(token='$token') repo_id_val = os.environ.get('DATASET_ID') file_path_val = '$file_path' file_name_val = '$file_name' backup_prefix_val = '$BACKUP_PREFIX' # Pass prefix correctly if not repo_id_val: print('[Python Upload] ERROR: DATASET_ID missing.'); sys.exit(1) if not os.path.exists(file_path_val): print(f'[Python Upload] ERROR: Backup file {file_path_val} not found.'); sys.exit(1) try: print(f'[Python Upload] Uploading: {file_path_val} to {repo_id_val} as {file_name_val}') upload_file( path_or_fileobj=file_path_val, path_in_repo=file_name_val, repo_id=repo_id_val, repo_type='dataset', token=os.environ.get('HF_TOKEN') # Explicitly pass token ) print(f'[Python Upload] Successfully uploaded {file_name_val}') manage_backups(api, repo_id_val, backup_prefix_val) # Pass prefix except Exception as e: import traceback print(f'[Python Upload] ERROR uploading file: {str(e)}') # traceback.print_exc() # Uncomment for full traceback if needed sys.exit(1) # Indicate failure " return $? # Return python exit code } # Python Function: Download Latest Backup download_latest_backup() { local token="$HF_TOKEN"; local repo_id="$DATASET_ID" local download_flag_file="/tmp/backup_restored.flag" # Flag file rm -f "$download_flag_file" # Ensure flag is removed initially echo "[Python Restore] Preparing to download from Dataset: $repo_id" python3 -c " import sys, os, tarfile, tempfile, shutil, subprocess, time from huggingface_hub import HfApi, hf_hub_download, list_repo_files print(f'[Python Restore] HF_TOKEN is set: {os.environ.get(\"HF_TOKEN\") is not None}') print(f'[Python Restore] DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}') flag_file = '$download_flag_file' api = HfApi(token='$token') repo_id_val = os.environ.get('DATASET_ID') if not repo_id_val: print('[Python Restore] ERROR: DATASET_ID missing.'); sys.exit(1) try: print(f'[Python Restore] Listing files in Dataset: {repo_id_val}') # Retry logic for listing files for attempt in range(3): try: files = list_repo_files(repo_id=repo_id_val, repo_type='dataset', token=os.environ.get('HF_TOKEN')) break # Success except Exception as list_err: print(f'[Python Restore] Error listing files (attempt {attempt+1}/3): {list_err}') if attempt == 2: raise time.sleep(5) backup_files = sorted([f for f in files if f.startswith('$BACKUP_PREFIX') and f.endswith('.tar.gz')]) if not backup_files: print('[Python Restore] No backup files found. Skipping restore.') sys.exit(0) # Success: No backup found is OK for script flow latest_backup = backup_files[-1] print(f'[Python Restore] Latest backup file found: {latest_backup}') with tempfile.TemporaryDirectory() as temp_dir: print(f'[Python Restore] Downloading {latest_backup} to {temp_dir}...') try: filepath = hf_hub_download(repo_id=repo_id_val, filename=latest_backup, repo_type='dataset', local_dir=temp_dir, token=os.environ.get('HF_TOKEN')) except Exception as download_error: print(f'[Python Restore] ERROR during download: {download_error}'); sys.exit(1) if filepath and os.path.exists(filepath): print(f'[Python Restore] Download successful: {filepath}') items_to_restore = ['cloudreve', 'cloudreve.db', 'config.ini'] # Files to restore target_dir = os.environ.get('CLOUDREVE_DIR', '/opt/cloudreve') # Get target dir from env os.makedirs(target_dir, exist_ok=True) print('[Python Restore] Contents before restore:'); subprocess.run(['ls', '-lA', target_dir], check=False) extract_temp_dir = os.path.join(temp_dir, 'extracted_backup'); os.makedirs(extract_temp_dir, exist_ok=True) print(f'[Python Restore] Extracting archive to {extract_temp_dir}') try: with tarfile.open(filepath, 'r:gz') as tar: tar.extractall(path=extract_temp_dir) print('[Python Restore] Extraction complete.') except Exception as extract_err: print(f'[Python Restore] ERROR during extraction: {extract_err}'); sys.exit(1) essential_files_present = True print(f'[Python Restore] Checking extracted items in {extract_temp_dir}:') for item in items_to_restore: extracted_path = os.path.join(extract_temp_dir, item) print(f'[Python Restore] Checking for: {extracted_path}') if not os.path.exists(extracted_path): print(f'[Python Restore] ERROR: Essential item "{item}" not found in backup.'); essential_files_present = False if not essential_files_present: print('[Python Restore] Aborting restore due to missing essential files.'); sys.exit(1) print(f'[Python Restore] Deleting existing items in {target_dir}...') for item in items_to_restore: target_path = os.path.join(target_dir, item) if os.path.exists(target_path): try: if os.path.isdir(target_path) and not os.path.islink(target_path): shutil.rmtree(target_path) else: os.remove(target_path) print(f'[Python Restore] Deleted: {target_path}') except OSError as e: print(f'[Python Restore] Error deleting {target_path}: {e}') print(f'[Python Restore] Moving extracted items to {target_dir}...') for item in items_to_restore: source = os.path.join(extract_temp_dir, item); target = os.path.join(target_dir, item) try: shutil.move(source, target); print(f'[Python Restore] Moved: {item}') except Exception as move_err: print(f'[Python Restore] Error moving {item}: {move_err}') print(f'[Python Restore] Successfully restored backup from {latest_backup}') print('[Python Restore] Contents after restore:'); subprocess.run(['ls', '-lA', target_dir], check=False) with open(flag_file, 'w') as f: f.write('restored') # Create flag file print(f'[Python Restore] Created restore flag file: {flag_file}') sys.exit(0) # Indicate successful restore else: print(f'[Python Restore] ERROR: Downloaded file path invalid or missing: {filepath}') sys.exit(1) except Exception as e: import traceback; print(f'[Python Restore] ERROR during download/restore: {str(e)}'); traceback.print_exc(); sys.exit(1) " return $? # Return python exit code } # --- Background Backup Manager Function --- manage_backups_background() { local restored_flag_path="$1" # Pass path to the flag file echo "[Backup Manager] Process started. Waiting for Cloudreve..." wait_for_cloudreve # Wait until Cloudreve port is open # Determine if initial backup is needed local perform_initial_backup=true if [ -f "$restored_flag_path" ]; then echo "[Backup Manager] Restore flag found. Skipping initial backup." perform_initial_backup=false else echo "[Backup Manager] No restore flag found. Performing initial backup." fi # Clean up flag file now that we've checked it rm -f "$restored_flag_path" # Perform initial backup if needed if [ "$perform_initial_backup" = true ]; then perform_backup_once "$CONFIG_FILE_PATH" "$DB_FILE_PATH" "$EXECUTABLE_PATH" "$CLOUDREVE_DIR" "$BACKUP_PREFIX" if [ $? -eq 0 ]; then echo "[Backup Manager] Initial backup completed." else echo "[Backup Manager] WARNING: Initial backup failed." # Decide if we should proceed anyway or exit? For now, proceed. fi fi # Start the periodic backup loop echo "[Backup Manager] Starting periodic backup loop..." while true; do local sync_interval=${SYNC_INTERVAL:-3600} # Default to 1 hour (3600 seconds) echo "[Backup Manager] Periodic Sync: Next cycle in ${sync_interval} seconds..." sleep "$sync_interval" echo "[Backup Manager] Periodic Sync: Starting cycle at $(date)" perform_backup_once "$CONFIG_FILE_PATH" "$DB_FILE_PATH" "$EXECUTABLE_PATH" "$CLOUDREVE_DIR" "$BACKUP_PREFIX" # Log completion, ignore errors for the loop to continue echo "[Backup Manager] Periodic Sync: Cycle finished." done } # --- Main Script Execution --- # Activate Python environment echo "Activating Python venv..." source /opt/venv/bin/activate # Define paths CLOUDREVE_DIR="/opt/cloudreve" BACKUP_PREFIX="cloudreve_backup" CONFIG_FILE_PATH="${CLOUDREVE_DIR}/config.ini" DB_FILE_PATH="${CLOUDREVE_DIR}/cloudreve.db" EXECUTABLE_PATH="${CLOUDREVE_DIR}/cloudreve" RESTORE_FLAG_FILE="/tmp/backup_restored.flag" # Define flag file path # Export CLOUDREVE_DIR for Python functions if needed export CLOUDREVE_DIR # 1. Attempt Initial Restore (sets flag file on success) echo "--- Step 1: Attempting Initial Restore ---" download_latest_backup # This function now creates $RESTORE_FLAG_FILE on success restore_exit_code=$? if [ $restore_exit_code -ne 0 ]; then echo "CRITICAL: Backup restoration attempt failed (Exit code: $restore_exit_code). Exiting." # Optional: Clean up flag file even on failure? rm -f "$RESTORE_FLAG_FILE" exit 1 elif [ -f "$RESTORE_FLAG_FILE" ]; then echo "--- Step 1 Result: Restore successful. ---" else echo "--- Step 1 Result: Restore skipped (no backup found). ---" fi # Flag file will be checked and removed by the background manager # 2. Start Background Backup Manager # It will wait for Cloudreve, check the flag file, do initial backup if needed, then loop. echo "--- Step 2: Starting Background Backup Manager ---" manage_backups_background "$RESTORE_FLAG_FILE" & # Pass flag file path backup_manager_pid=$! # 3. Start Cloudreve Main Process using exec # This handles first run (init+exit) and normal runs. # It becomes the container's main process. echo "--- Step 3: Starting Cloudreve Application (using exec) ---" exec /opt/cloudreve/cloudreve -c "$CONFIG_FILE_PATH" # --- Code below only runs if exec fails --- exec_failed_code=$? echo "CRITICAL: Failed to execute Cloudreve (Exit code: $exec_failed_code). Terminating background jobs..." # Attempt to kill the background backup manager if exec failed kill $backup_manager_pid 2>/dev/null # Attempt to kill Aria2 if it was started by CMD? This is harder to manage reliably here. exit $exec_failed_code