flzta commited on
Commit
6fa1e07
·
verified ·
1 Parent(s): aaa89a6

Update sync_data.sh

Browse files
Files changed (1) hide show
  1. sync_data.sh +216 -221
sync_data.sh CHANGED
@@ -1,336 +1,331 @@
1
  #!/bin/bash
2
 
 
 
3
  # Function to check if Cloudreve is running by checking the port
4
  wait_for_cloudreve() {
5
- echo "Waiting for Cloudreve service to become available on port 5212..."
6
- # Loop until nc successfully connects (-z checks without sending data)
7
  while ! nc -z 127.0.0.1 5212; do
8
- echo "Cloudreve not ready yet (port 5212 not open), sleeping for 5 seconds..."
9
  sleep 5
10
  done
11
- echo "Cloudreve service detected on port 5212."
12
  }
13
 
14
  # Function to perform a single backup action
15
  perform_backup_once() {
16
- echo "Performing single backup operation..."
17
  local config_path="$1"
18
  local db_path="$2"
19
  local exe_path="$3"
20
  local cloudreve_dir="$4"
21
  local backup_prefix="$5"
22
 
 
 
 
23
  if [ ! -f "$config_path" ] || [ ! -f "$db_path" ] || [ ! -f "$exe_path" ]; then
24
- echo "WARN: Essential files missing for backup. Skipping this backup."
25
  return 1
26
  fi
27
 
28
  local timestamp=$(date +%Y%m%d_%H%M%S)
29
  local backup_file="${backup_prefix}_${timestamp}.tar.gz"
30
- local backup_path="/tmp/${backup_file}" # Use /tmp for temporary files
31
 
32
- echo "Compressing Cloudreve data (executable, db, config) to: $backup_path"
33
  tar -czf "$backup_path" -C "$cloudreve_dir" \
34
- $(basename "$exe_path") \
35
- $(basename "$db_path") \
36
- $(basename "$config_path")
 
37
 
38
  if [ -s "$backup_path" ]; then
39
- echo "Compression complete. File size: $(ls -lh "$backup_path" | awk '{print $5}')"
40
- echo "Uploading backup to HuggingFace..."
41
- # Call the previously defined upload_backup function
42
  upload_backup "$backup_path" "${backup_file}"
43
  local upload_status=$?
44
  if [ $upload_status -ne 0 ]; then
45
- echo "Backup upload failed. Keeping local archive: $backup_path"
46
  return 1
47
  else
48
- echo "Upload successful. Removing local archive."
49
  rm -f "$backup_path"
50
  return 0
51
  fi
52
  else
53
- echo "Compression failed or created an empty file. Skipping upload."
54
- rm -f "$backup_path" # Remove potentially empty/corrupt file
55
  return 1
56
  fi
57
  }
58
 
59
-
60
- # --- Start Script Execution ---
61
-
62
- # Check Hugging Face Token and Dataset ID environment variables
63
- if [[ -z "$HF_TOKEN" ]] || [[ -z "$DATASET_ID" ]]; then
64
- echo "Starting without backup functionality - missing HF_TOKEN or DATASET_ID"
65
- # Ensure Aria2 starts if needed, even without backup/restore
66
- # Example (adjust options as needed from your Dockerfile CMD):
67
- # aria2c --daemon=true [OTHER_OPTIONS] &
68
- exec /opt/cloudreve/cloudreve -c /opt/cloudreve/config.ini
69
- exit 0
70
- fi
71
-
72
- # Activate Python virtual environment
73
- echo "Activating Python venv..."
74
- source /opt/venv/bin/activate
75
-
76
- # Define Cloudreve paths and backup prefix
77
- CLOUDREVE_DIR="/opt/cloudreve"
78
- BACKUP_PREFIX="cloudreve_backup"
79
- CONFIG_FILE_PATH="/opt/cloudreve/config.ini"
80
- DB_FILE_PATH="/opt/cloudreve/cloudreve.db"
81
- EXECUTABLE_PATH="/opt/cloudreve/cloudreve"
82
-
83
-
84
- # --- Python Function Definitions (Ensure these are complete) ---
85
 
86
  # Python Function: Upload Backup
87
  upload_backup() {
88
- local file_path="$1"
89
- local file_name="$2"
90
- local token="$HF_TOKEN"
91
- local repo_id="$DATASET_ID"
92
-
93
- echo "Preparing to upload backup file: $file_path as $file_name to Dataset: $repo_id"
94
-
95
- # Use python3 -c "..." with the full Python code block here
96
- # (Using placeholder for brevity, replace with your actual Python code)
97
  python3 -c "
98
- from huggingface_hub import HfApi
99
- import sys, os
100
- print(f'HF_TOKEN is set: {os.environ.get(\"HF_TOKEN\") is not None}')
101
- print(f'DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}')
102
- def manage_backups(api, repo_id_val, max_files=50): # Increased max_files
103
- print('Managing old backups...')
 
104
  try:
105
- files = api.list_repo_files(repo_id=repo_id_val, repo_type='dataset')
106
- backup_files = sorted([f for f in files if f.startswith('$BACKUP_PREFIX') and f.endswith('.tar.gz')])
 
 
 
 
 
 
 
 
 
 
107
  if len(backup_files) >= max_files:
108
- print(f'Found {len(backup_files)} backup files, maximum allowed is {max_files}.')
109
- files_to_delete = backup_files[:len(backup_files) - max_files + 1]
110
- print(f'Will delete {len(files_to_delete)} old backups.')
111
  for file_to_delete in files_to_delete:
112
  try:
113
- print(f'Deleting old backup: {file_to_delete}')
114
- api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id_val, repo_type='dataset', token=os.environ.get('HF_TOKEN'))
115
- print(f'Successfully deleted: {file_to_delete}')
116
  except Exception as e:
117
- print(f'Error deleting {file_to_delete}: {str(e)}')
118
  else:
119
- print(f'Number of backup files ({len(backup_files)}) is within the limit ({max_files}).')
120
  except Exception as e:
121
- print(f'Error listing or managing backups: {e}')
122
 
123
  api = HfApi(token='$token')
 
 
 
 
 
 
 
 
124
  try:
125
- repo_id_val = os.environ.get('DATASET_ID')
126
- if not repo_id_val: raise ValueError('DATASET_ID environment variable is not set.')
127
- print(f'Uploading file: $file_path to {repo_id_val} as $file_name')
128
- api.upload_file(path_or_fileobj='$file_path', path_in_repo='$file_name', repo_id=repo_id_val, repo_type='dataset', token=os.environ.get('HF_TOKEN'))
129
- print(f'Successfully uploaded $file_name')
130
- manage_backups(api, repo_id_val)
 
 
 
 
131
  except Exception as e:
132
- print(f'Error uploading file: {str(e)}')
 
 
133
  sys.exit(1) # Indicate failure
134
  "
135
- # Capture python exit code
136
- return $?
137
  }
138
 
139
  # Python Function: Download Latest Backup
140
  download_latest_backup() {
141
- local token="$HF_TOKEN"
142
- local repo_id="$DATASET_ID"
143
  local download_flag_file="/tmp/backup_restored.flag" # Flag file
 
 
144
 
145
- # Remove any previous flag file
146
- rm -f "$download_flag_file"
147
-
148
- echo "Preparing to download the latest backup from Dataset: $repo_id"
149
-
150
- # Use python3 -c "..." with the full Python code block here
151
- # (Using placeholder for brevity, replace with your actual Python code)
152
  python3 -c "
153
- from huggingface_hub import HfApi, hf_hub_download
154
- import sys, os, tarfile, tempfile, shutil, subprocess
155
- print(f'HF_TOKEN is set: {os.environ.get(\"HF_TOKEN\") is not None}')
156
- print(f'DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}')
157
- flag_file = '$download_flag_file' # Get flag file path
158
 
159
  api = HfApi(token='$token')
 
 
 
160
  try:
161
- repo_id_val = os.environ.get('DATASET_ID')
162
- if not repo_id_val: raise ValueError('DATASET_ID environment variable is not set.')
 
 
 
 
 
 
 
 
163
 
164
- print(f'Listing files in Dataset: {repo_id_val}')
165
- files = api.list_repo_files(repo_id=repo_id_val, repo_type='dataset')
166
  backup_files = sorted([f for f in files if f.startswith('$BACKUP_PREFIX') and f.endswith('.tar.gz')])
167
 
168
  if not backup_files:
169
- print('No backup files found in the Dataset. Skipping restore.')
170
- sys.exit(0) # Success: No backup found is not an error for the script flow
171
 
172
  latest_backup = backup_files[-1]
173
- print(f'Latest backup file found: {latest_backup}')
174
 
175
  with tempfile.TemporaryDirectory() as temp_dir:
176
- print(f'Downloading {latest_backup} to temporary directory {temp_dir}...')
177
  try:
178
  filepath = hf_hub_download(repo_id=repo_id_val, filename=latest_backup, repo_type='dataset', local_dir=temp_dir, token=os.environ.get('HF_TOKEN'))
179
- except Exception as download_error:
180
- print(f'Error during hf_hub_download: {download_error}'); sys.exit(1)
181
 
182
  if filepath and os.path.exists(filepath):
183
- print(f'Successfully downloaded backup to temporary directory: {filepath}')
184
- items_to_restore = ['cloudreve', 'cloudreve.db', 'config.ini']
185
- os.makedirs(\"$CLOUDREVE_DIR\", exist_ok=True)
186
- print('Listing contents before restore:'); subprocess.run(['ls', '-lA', \"$CLOUDREVE_DIR\"], check=False)
 
187
 
188
  extract_temp_dir = os.path.join(temp_dir, 'extracted_backup'); os.makedirs(extract_temp_dir, exist_ok=True)
189
- print(f'Extracting backup archive: {filepath} to {extract_temp_dir}')
190
  try:
191
- with tarfile.open(filepath, 'r:gz') as tar: tar.extractall(extract_temp_dir)
192
- print('Extraction complete.')
193
- except Exception as extract_err: print(f'Error during extraction: {extract_err}'); sys.exit(1)
194
 
195
  essential_files_present = True
 
196
  for item in items_to_restore:
197
- extracted_item_path = os.path.join(extract_temp_dir, item)
198
- if not os.path.exists(extracted_item_path):
199
- print(f'Error: Essential item "{item}" not found in extracted backup at {extracted_item_path}. Aborting restore.')
200
- essential_files_present = False; # break # Keep checking all missing items
201
  if not essential_files_present:
202
- print('Listing contents of extraction directory for debugging:'); subprocess.run(['ls', '-lA', extract_temp_dir], check=False)
203
- sys.exit(1) # Abort if essential files are missing
204
 
205
- print(f'Deleting existing items in $CLOUDREVE_DIR before restoring...')
206
  for item in items_to_restore:
207
- target_path = os.path.join(\"$CLOUDREVE_DIR\", item)
208
  if os.path.exists(target_path):
209
  try:
210
  if os.path.isdir(target_path) and not os.path.islink(target_path): shutil.rmtree(target_path)
211
  else: os.remove(target_path)
212
- except OSError as e: print(f'Error deleting {target_path}: {e}. Continuing...')
 
213
 
214
- print(f'Moving extracted items from {extract_temp_dir} to $CLOUDREVE_DIR...')
215
  for item in items_to_restore:
216
- source_path = os.path.join(extract_temp_dir, item)
217
- if os.path.exists(source_path):
218
- target_path = os.path.join(\"$CLOUDREVE_DIR\", item)
219
- try: shutil.move(source_path, target_path)
220
- except Exception as move_err: print(f'Error moving {item}: {move_err}') # Continue for now
221
- else: print(f'Warning: Source item {item} disappeared before move.')
222
-
223
- print(f'Successfully restored backup from {latest_backup}')
224
- print('Listing contents after restore:'); subprocess.run(['ls', '-lA', \"$CLOUDREVE_DIR\"], check=False)
225
- # Create the flag file only on successful restore
226
- with open(flag_file, 'w') as f: f.write('restored')
227
- print(f'Created restore flag file: {flag_file}')
228
- sys.exit(0) # Indicate successful restore completion
229
  else:
230
- print(f'Error: Downloaded file path "{filepath}" does not exist or download failed.')
231
  sys.exit(1)
232
 
233
- except ValueError as ve: print(f'Configuration Error: {ve}'); sys.exit(1)
234
- except Exception as e: import traceback; print(f'Error during backup download/restore: {str(e)}'); traceback.print_exc(); sys.exit(1)
235
  "
236
- # Capture python exit code
237
- return $?
238
  }
239
 
240
- # --- Periodic Sync Function (Loop only) ---
241
- sync_data_loop() {
242
- echo "Starting Periodic Background Sync Loop..."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
243
  while true; do
244
- # Wait for the defined interval
245
- local sync_interval=${SYNC_INTERVAL:-3600} # Default to 1 hour
246
- echo "Periodic Sync: Next check in ${sync_interval} seconds..."
247
- sleep $sync_interval
248
 
249
- # Perform the backup
250
- echo "Periodic Sync: Starting backup cycle at $(date)"
251
  perform_backup_once "$CONFIG_FILE_PATH" "$DB_FILE_PATH" "$EXECUTABLE_PATH" "$CLOUDREVE_DIR" "$BACKUP_PREFIX"
252
- # We don't necessarily need to exit the loop if one backup fails
 
253
  done
254
  }
255
 
256
 
257
- # --- Main Execution Logic ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
258
 
259
- # 1. Attempt Initial Restore
260
  echo "--- Step 1: Attempting Initial Restore ---"
261
- download_latest_backup
262
  restore_exit_code=$?
263
- restore_flag_file="/tmp/backup_restored.flag"
264
 
265
- # Check restore outcome
266
- backup_was_restored=false
267
  if [ $restore_exit_code -ne 0 ]; then
268
- echo "CRITICAL: Backup restoration attempt failed with error (Exit code: $restore_exit_code). Exiting."
 
 
269
  exit 1
270
- elif [ -f "$restore_flag_file" ]; then
271
- echo "Restore successful: Backup found and restored."
272
- backup_was_restored=true
273
- else
274
- echo "Restore skipped: No backup file found in the dataset."
275
- backup_was_restored=false
276
- fi
277
- # Clean up flag file regardless
278
- rm -f "$restore_flag_file"
279
-
280
-
281
- # 2. Initial Cloudreve Run (If necessary)
282
- echo "--- Step 2: Checking for Initial Config ---"
283
- if [ ! -f "$CONFIG_FILE_PATH" ]; then
284
- echo "Config file ($CONFIG_FILE_PATH) not found. Running Cloudreve once to generate initial config..."
285
- /opt/cloudreve/cloudreve -c "$CONFIG_FILE_PATH"
286
- if [ ! -f "$CONFIG_FILE_PATH" ]; then
287
- echo "CRITICAL: Cloudreve failed to create initial config file. Exiting."
288
- exit 1
289
- else
290
- echo "Initial config file created by first run. Check logs for credentials if needed."
291
- fi
292
- fi
293
-
294
- # 3. Start Cloudreve Main Process (in background temporarily to allow checks)
295
- echo "--- Step 3: Starting Cloudreve Main Process (background initially) ---"
296
- /opt/cloudreve/cloudreve -c "$CONFIG_FILE_PATH" &
297
- cloudreve_pid=$!
298
-
299
- # 4. Wait for Cloudreve to be Ready
300
- echo "--- Step 4: Waiting for Cloudreve Service ---"
301
- wait_for_cloudreve # Use the function defined earlier
302
-
303
- # 5. Perform Initial Backup (If No Restore Occurred)
304
- echo "--- Step 5: Checking for Initial Backup ---"
305
- if [ "$backup_was_restored" = false ]; then
306
- echo "No backup was restored, performing initial backup of current state..."
307
- perform_backup_once "$CONFIG_FILE_PATH" "$DB_FILE_PATH" "$EXECUTABLE_PATH" "$CLOUDREVE_DIR" "$BACKUP_PREFIX"
308
- if [ $? -ne 0 ]; then
309
- echo "Warning: Initial backup failed."
310
- # Decide whether to continue or exit if initial backup fails
311
- else
312
- echo "Initial backup completed successfully."
313
- fi
314
  else
315
- echo "Backup was restored, skipping initial backup."
316
  fi
317
-
318
- # 6. Start Periodic Sync Loop in Background
319
- echo "--- Step 6: Starting Periodic Sync Loop ---"
320
- sync_data_loop &
321
- sync_loop_pid=$!
322
-
323
- # 7. Bring Cloudreve to Foreground / Wait for it
324
- # Since Cloudreve was started in the background, we now wait for it.
325
- # If Cloudreve exits, this script will also exit.
326
- echo "--- Step 7: Cloudreve is running. Monitoring PID: $cloudreve_pid ---"
327
- wait $cloudreve_pid
328
- exit_code=$?
329
- echo "Cloudreve process exited with code $exit_code."
330
-
331
- # Optional: attempt to clean up background sync loop if Cloudreve exits unexpectedly
332
- echo "Attempting to terminate background sync loop (PID: $sync_loop_pid)..."
333
- kill $sync_loop_pid 2>/dev/null
334
- wait $sync_loop_pid 2>/dev/null # Wait briefly for it to terminate
335
-
336
- exit $exit_code
 
 
1
  #!/bin/bash
2
 
3
+ # --- Helper Functions ---
4
+
5
  # Function to check if Cloudreve is running by checking the port
6
  wait_for_cloudreve() {
7
+ echo "[Backup Manager] Waiting for Cloudreve service (port 5212)..."
 
8
  while ! nc -z 127.0.0.1 5212; do
9
+ echo "[Backup Manager] Cloudreve not ready yet, sleeping 5s..."
10
  sleep 5
11
  done
12
+ echo "[Backup Manager] Cloudreve service detected."
13
  }
14
 
15
  # Function to perform a single backup action
16
  perform_backup_once() {
17
+ echo "[Backup Action] Performing single backup..."
18
  local config_path="$1"
19
  local db_path="$2"
20
  local exe_path="$3"
21
  local cloudreve_dir="$4"
22
  local backup_prefix="$5"
23
 
24
+ # Wait brief moment to ensure files are stable after service start
25
+ sleep 5
26
+
27
  if [ ! -f "$config_path" ] || [ ! -f "$db_path" ] || [ ! -f "$exe_path" ]; then
28
+ echo "[Backup Action] WARN: Essential files missing for backup ($config_path, $db_path, $exe_path). Skipping."
29
  return 1
30
  fi
31
 
32
  local timestamp=$(date +%Y%m%d_%H%M%S)
33
  local backup_file="${backup_prefix}_${timestamp}.tar.gz"
34
+ local backup_path="/tmp/${backup_file}"
35
 
36
+ echo "[Backup Action] Compressing data (exec, db, config) to: $backup_path"
37
  tar -czf "$backup_path" -C "$cloudreve_dir" \
38
+ "$(basename "$exe_path")" \
39
+ "$(basename "$db_path")" \
40
+ "$(basename "$config_path")" \
41
+ || { echo "[Backup Action] ERROR: tar command failed."; rm -f "$backup_path"; return 1; } # Add error check
42
 
43
  if [ -s "$backup_path" ]; then
44
+ echo "[Backup Action] Compression complete. Size: $(ls -lh "$backup_path" | awk '{print $5}')"
45
+ echo "[Backup Action] Uploading backup to HuggingFace..."
46
+ # Call the Python upload_backup function
47
  upload_backup "$backup_path" "${backup_file}"
48
  local upload_status=$?
49
  if [ $upload_status -ne 0 ]; then
50
+ echo "[Backup Action] ERROR: Backup upload failed. Keeping local archive: $backup_path"
51
  return 1
52
  else
53
+ echo "[Backup Action] Upload successful. Removing local archive."
54
  rm -f "$backup_path"
55
  return 0
56
  fi
57
  else
58
+ echo "[Backup Action] ERROR: Compression failed or created empty file. Skipping upload."
59
+ rm -f "$backup_path"
60
  return 1
61
  fi
62
  }
63
 
64
+ # --- Python Function Definitions (Ensure these are complete and correct) ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
 
66
  # Python Function: Upload Backup
67
  upload_backup() {
68
+ local file_path="$1"; local file_name="$2"; local token="$HF_TOKEN"; local repo_id="$DATASET_ID"
69
+ echo "[Python Upload] Preparing: $file_path as $file_name to $repo_id"
 
 
 
 
 
 
 
70
  python3 -c "
71
+ import sys, os, time
72
+ from huggingface_hub import HfApi, list_repo_files, delete_file, upload_file
73
+ print(f'[Python Upload] HF_TOKEN is set: {os.environ.get(\"HF_TOKEN\") is not None}')
74
+ print(f'[Python Upload] DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}')
75
+
76
+ def manage_backups(api, repo_id_val, backup_prefix_val, max_files=50): # Increased max_files
77
+ print('[Python Upload] Managing old backups...')
78
  try:
79
+ # Retry logic for listing files, as sometimes HF API might be slow/flaky
80
+ for attempt in range(3):
81
+ try:
82
+ files = list_repo_files(repo_id=repo_id_val, repo_type='dataset', token=os.environ.get('HF_TOKEN'))
83
+ break # Success
84
+ except Exception as list_err:
85
+ print(f'[Python Upload] Error listing files (attempt {attempt+1}/3): {list_err}')
86
+ if attempt == 2: raise # Raise after last attempt
87
+ time.sleep(5) # Wait before retrying
88
+
89
+ backup_files = sorted([f for f in files if f.startswith(backup_prefix_val) and f.endswith('.tar.gz')])
90
+ print(f'[Python Upload] Found {len(backup_files)} existing backup files.')
91
  if len(backup_files) >= max_files:
92
+ num_to_delete = len(backup_files) - max_files + 1
93
+ print(f'[Python Upload] Max backups ({max_files}) reached. Need to delete {num_to_delete} oldest backups.')
94
+ files_to_delete = backup_files[:num_to_delete]
95
  for file_to_delete in files_to_delete:
96
  try:
97
+ print(f'[Python Upload] Deleting old backup: {file_to_delete}')
98
+ delete_file(path_in_repo=file_to_delete, repo_id=repo_id_val, repo_type='dataset', token=os.environ.get('HF_TOKEN'))
99
+ print(f'[Python Upload] Successfully deleted: {file_to_delete}')
100
  except Exception as e:
101
+ print(f'[Python Upload] ERROR deleting {file_to_delete}: {str(e)}') # Log error but continue
102
  else:
103
+ print(f'[Python Upload] Number of backups ({len(backup_files)}) is within the limit ({max_files}).')
104
  except Exception as e:
105
+ print(f'[Python Upload] ERROR during backup management: {e}')
106
 
107
  api = HfApi(token='$token')
108
+ repo_id_val = os.environ.get('DATASET_ID')
109
+ file_path_val = '$file_path'
110
+ file_name_val = '$file_name'
111
+ backup_prefix_val = '$BACKUP_PREFIX' # Pass prefix correctly
112
+
113
+ if not repo_id_val: print('[Python Upload] ERROR: DATASET_ID missing.'); sys.exit(1)
114
+ if not os.path.exists(file_path_val): print(f'[Python Upload] ERROR: Backup file {file_path_val} not found.'); sys.exit(1)
115
+
116
  try:
117
+ print(f'[Python Upload] Uploading: {file_path_val} to {repo_id_val} as {file_name_val}')
118
+ upload_file(
119
+ path_or_fileobj=file_path_val,
120
+ path_in_repo=file_name_val,
121
+ repo_id=repo_id_val,
122
+ repo_type='dataset',
123
+ token=os.environ.get('HF_TOKEN') # Explicitly pass token
124
+ )
125
+ print(f'[Python Upload] Successfully uploaded {file_name_val}')
126
+ manage_backups(api, repo_id_val, backup_prefix_val) # Pass prefix
127
  except Exception as e:
128
+ import traceback
129
+ print(f'[Python Upload] ERROR uploading file: {str(e)}')
130
+ # traceback.print_exc() # Uncomment for full traceback if needed
131
  sys.exit(1) # Indicate failure
132
  "
133
+ return $? # Return python exit code
 
134
  }
135
 
136
  # Python Function: Download Latest Backup
137
  download_latest_backup() {
138
+ local token="$HF_TOKEN"; local repo_id="$DATASET_ID"
 
139
  local download_flag_file="/tmp/backup_restored.flag" # Flag file
140
+ rm -f "$download_flag_file" # Ensure flag is removed initially
141
+ echo "[Python Restore] Preparing to download from Dataset: $repo_id"
142
 
 
 
 
 
 
 
 
143
  python3 -c "
144
+ import sys, os, tarfile, tempfile, shutil, subprocess, time
145
+ from huggingface_hub import HfApi, hf_hub_download, list_repo_files
146
+ print(f'[Python Restore] HF_TOKEN is set: {os.environ.get(\"HF_TOKEN\") is not None}')
147
+ print(f'[Python Restore] DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}')
148
+ flag_file = '$download_flag_file'
149
 
150
  api = HfApi(token='$token')
151
+ repo_id_val = os.environ.get('DATASET_ID')
152
+ if not repo_id_val: print('[Python Restore] ERROR: DATASET_ID missing.'); sys.exit(1)
153
+
154
  try:
155
+ print(f'[Python Restore] Listing files in Dataset: {repo_id_val}')
156
+ # Retry logic for listing files
157
+ for attempt in range(3):
158
+ try:
159
+ files = list_repo_files(repo_id=repo_id_val, repo_type='dataset', token=os.environ.get('HF_TOKEN'))
160
+ break # Success
161
+ except Exception as list_err:
162
+ print(f'[Python Restore] Error listing files (attempt {attempt+1}/3): {list_err}')
163
+ if attempt == 2: raise
164
+ time.sleep(5)
165
 
 
 
166
  backup_files = sorted([f for f in files if f.startswith('$BACKUP_PREFIX') and f.endswith('.tar.gz')])
167
 
168
  if not backup_files:
169
+ print('[Python Restore] No backup files found. Skipping restore.')
170
+ sys.exit(0) # Success: No backup found is OK for script flow
171
 
172
  latest_backup = backup_files[-1]
173
+ print(f'[Python Restore] Latest backup file found: {latest_backup}')
174
 
175
  with tempfile.TemporaryDirectory() as temp_dir:
176
+ print(f'[Python Restore] Downloading {latest_backup} to {temp_dir}...')
177
  try:
178
  filepath = hf_hub_download(repo_id=repo_id_val, filename=latest_backup, repo_type='dataset', local_dir=temp_dir, token=os.environ.get('HF_TOKEN'))
179
+ except Exception as download_error: print(f'[Python Restore] ERROR during download: {download_error}'); sys.exit(1)
 
180
 
181
  if filepath and os.path.exists(filepath):
182
+ print(f'[Python Restore] Download successful: {filepath}')
183
+ items_to_restore = ['cloudreve', 'cloudreve.db', 'config.ini'] # Files to restore
184
+ target_dir = os.environ.get('CLOUDREVE_DIR', '/opt/cloudreve') # Get target dir from env
185
+ os.makedirs(target_dir, exist_ok=True)
186
+ print('[Python Restore] Contents before restore:'); subprocess.run(['ls', '-lA', target_dir], check=False)
187
 
188
  extract_temp_dir = os.path.join(temp_dir, 'extracted_backup'); os.makedirs(extract_temp_dir, exist_ok=True)
189
+ print(f'[Python Restore] Extracting archive to {extract_temp_dir}')
190
  try:
191
+ with tarfile.open(filepath, 'r:gz') as tar: tar.extractall(path=extract_temp_dir)
192
+ print('[Python Restore] Extraction complete.')
193
+ except Exception as extract_err: print(f'[Python Restore] ERROR during extraction: {extract_err}'); sys.exit(1)
194
 
195
  essential_files_present = True
196
+ print(f'[Python Restore] Checking extracted items in {extract_temp_dir}:')
197
  for item in items_to_restore:
198
+ extracted_path = os.path.join(extract_temp_dir, item)
199
+ print(f'[Python Restore] Checking for: {extracted_path}')
200
+ if not os.path.exists(extracted_path):
201
+ print(f'[Python Restore] ERROR: Essential item "{item}" not found in backup.'); essential_files_present = False
202
  if not essential_files_present:
203
+ print('[Python Restore] Aborting restore due to missing essential files.'); sys.exit(1)
 
204
 
205
+ print(f'[Python Restore] Deleting existing items in {target_dir}...')
206
  for item in items_to_restore:
207
+ target_path = os.path.join(target_dir, item)
208
  if os.path.exists(target_path):
209
  try:
210
  if os.path.isdir(target_path) and not os.path.islink(target_path): shutil.rmtree(target_path)
211
  else: os.remove(target_path)
212
+ print(f'[Python Restore] Deleted: {target_path}')
213
+ except OSError as e: print(f'[Python Restore] Error deleting {target_path}: {e}')
214
 
215
+ print(f'[Python Restore] Moving extracted items to {target_dir}...')
216
  for item in items_to_restore:
217
+ source = os.path.join(extract_temp_dir, item); target = os.path.join(target_dir, item)
218
+ try: shutil.move(source, target); print(f'[Python Restore] Moved: {item}')
219
+ except Exception as move_err: print(f'[Python Restore] Error moving {item}: {move_err}')
220
+
221
+ print(f'[Python Restore] Successfully restored backup from {latest_backup}')
222
+ print('[Python Restore] Contents after restore:'); subprocess.run(['ls', '-lA', target_dir], check=False)
223
+ with open(flag_file, 'w') as f: f.write('restored') # Create flag file
224
+ print(f'[Python Restore] Created restore flag file: {flag_file}')
225
+ sys.exit(0) # Indicate successful restore
 
 
 
 
226
  else:
227
+ print(f'[Python Restore] ERROR: Downloaded file path invalid or missing: {filepath}')
228
  sys.exit(1)
229
 
230
+ except Exception as e: import traceback; print(f'[Python Restore] ERROR during download/restore: {str(e)}'); traceback.print_exc(); sys.exit(1)
 
231
  "
232
+ return $? # Return python exit code
 
233
  }
234
 
235
+ # --- Background Backup Manager Function ---
236
+ manage_backups_background() {
237
+ local restored_flag_path="$1" # Pass path to the flag file
238
+
239
+ echo "[Backup Manager] Process started. Waiting for Cloudreve..."
240
+ wait_for_cloudreve # Wait until Cloudreve port is open
241
+
242
+ # Determine if initial backup is needed
243
+ local perform_initial_backup=true
244
+ if [ -f "$restored_flag_path" ]; then
245
+ echo "[Backup Manager] Restore flag found. Skipping initial backup."
246
+ perform_initial_backup=false
247
+ else
248
+ echo "[Backup Manager] No restore flag found. Performing initial backup."
249
+ fi
250
+ # Clean up flag file now that we've checked it
251
+ rm -f "$restored_flag_path"
252
+
253
+ # Perform initial backup if needed
254
+ if [ "$perform_initial_backup" = true ]; then
255
+ perform_backup_once "$CONFIG_FILE_PATH" "$DB_FILE_PATH" "$EXECUTABLE_PATH" "$CLOUDREVE_DIR" "$BACKUP_PREFIX"
256
+ if [ $? -eq 0 ]; then
257
+ echo "[Backup Manager] Initial backup completed."
258
+ else
259
+ echo "[Backup Manager] WARNING: Initial backup failed."
260
+ # Decide if we should proceed anyway or exit? For now, proceed.
261
+ fi
262
+ fi
263
+
264
+ # Start the periodic backup loop
265
+ echo "[Backup Manager] Starting periodic backup loop..."
266
  while true; do
267
+ local sync_interval=${SYNC_INTERVAL:-3600} # Default to 1 hour (3600 seconds)
268
+ echo "[Backup Manager] Periodic Sync: Next cycle in ${sync_interval} seconds..."
269
+ sleep "$sync_interval"
 
270
 
271
+ echo "[Backup Manager] Periodic Sync: Starting cycle at $(date)"
 
272
  perform_backup_once "$CONFIG_FILE_PATH" "$DB_FILE_PATH" "$EXECUTABLE_PATH" "$CLOUDREVE_DIR" "$BACKUP_PREFIX"
273
+ # Log completion, ignore errors for the loop to continue
274
+ echo "[Backup Manager] Periodic Sync: Cycle finished."
275
  done
276
  }
277
 
278
 
279
+ # --- Main Script Execution ---
280
+
281
+ # Activate Python environment
282
+ echo "Activating Python venv..."
283
+ source /opt/venv/bin/activate
284
+
285
+ # Define paths
286
+ CLOUDREVE_DIR="/opt/cloudreve"
287
+ BACKUP_PREFIX="cloudreve_backup"
288
+ CONFIG_FILE_PATH="${CLOUDREVE_DIR}/config.ini"
289
+ DB_FILE_PATH="${CLOUDREVE_DIR}/cloudreve.db"
290
+ EXECUTABLE_PATH="${CLOUDREVE_DIR}/cloudreve"
291
+ RESTORE_FLAG_FILE="/tmp/backup_restored.flag" # Define flag file path
292
+
293
+ # Export CLOUDREVE_DIR for Python functions if needed
294
+ export CLOUDREVE_DIR
295
 
296
+ # 1. Attempt Initial Restore (sets flag file on success)
297
  echo "--- Step 1: Attempting Initial Restore ---"
298
+ download_latest_backup # This function now creates $RESTORE_FLAG_FILE on success
299
  restore_exit_code=$?
 
300
 
 
 
301
  if [ $restore_exit_code -ne 0 ]; then
302
+ echo "CRITICAL: Backup restoration attempt failed (Exit code: $restore_exit_code). Exiting."
303
+ # Optional: Clean up flag file even on failure?
304
+ rm -f "$RESTORE_FLAG_FILE"
305
  exit 1
306
+ elif [ -f "$RESTORE_FLAG_FILE" ]; then
307
+ echo "--- Step 1 Result: Restore successful. ---"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
308
  else
309
+ echo "--- Step 1 Result: Restore skipped (no backup found). ---"
310
  fi
311
+ # Flag file will be checked and removed by the background manager
312
+
313
+ # 2. Start Background Backup Manager
314
+ # It will wait for Cloudreve, check the flag file, do initial backup if needed, then loop.
315
+ echo "--- Step 2: Starting Background Backup Manager ---"
316
+ manage_backups_background "$RESTORE_FLAG_FILE" & # Pass flag file path
317
+ backup_manager_pid=$!
318
+
319
+ # 3. Start Cloudreve Main Process using exec
320
+ # This handles first run (init+exit) and normal runs.
321
+ # It becomes the container's main process.
322
+ echo "--- Step 3: Starting Cloudreve Application (using exec) ---"
323
+ exec /opt/cloudreve/cloudreve -c "$CONFIG_FILE_PATH"
324
+
325
+ # --- Code below only runs if exec fails ---
326
+ exec_failed_code=$?
327
+ echo "CRITICAL: Failed to execute Cloudreve (Exit code: $exec_failed_code). Terminating background jobs..."
328
+ # Attempt to kill the background backup manager if exec failed
329
+ kill $backup_manager_pid 2>/dev/null
330
+ # Attempt to kill Aria2 if it was started by CMD? This is harder to manage reliably here.
331
+ exit $exec_failed_code