flzta commited on
Commit
aaa89a6
·
verified ·
1 Parent(s): 3c591df

Update sync_data.sh

Browse files
Files changed (1) hide show
  1. sync_data.sh +226 -246
sync_data.sh CHANGED
@@ -1,356 +1,336 @@
1
  #!/bin/bash
2
 
3
- # 检查 Hugging Face Token Dataset ID 环境变量
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  if [[ -z "$HF_TOKEN" ]] || [[ -z "$DATASET_ID" ]]; then
5
- echo "Starting Cloudreve without backup/restore functionality - missing HF_TOKEN or DATASET_ID"
6
- # 直接启动 Cloudreve 作为主进程
7
- echo "Starting Cloudreve directly..."
8
- # 假设无备份时仍需 Aria2:
9
- # 注意:如果不需要 Aria2,应移除下面的 Aria2 启动或确保 Dockerfile 的 CMD 也移除了 Aria2
10
- # aria2c [YOUR_ARIA2_OPTIONS_HERE] & # 根据需要添加 Aria2 参数
11
  exec /opt/cloudreve/cloudreve -c /opt/cloudreve/config.ini
12
- exit 0 # exec 通常不会返回,但加上 exit 0 以防万一
13
  fi
14
 
15
- # 激活 Python 虚拟环境
16
  echo "Activating Python venv..."
17
  source /opt/venv/bin/activate
18
 
19
- # 定义 Cloudreve 主程序目录 备份文件前缀
20
  CLOUDREVE_DIR="/opt/cloudreve"
21
  BACKUP_PREFIX="cloudreve_backup"
22
  CONFIG_FILE_PATH="/opt/cloudreve/config.ini"
23
  DB_FILE_PATH="/opt/cloudreve/cloudreve.db"
24
  EXECUTABLE_PATH="/opt/cloudreve/cloudreve"
25
 
26
- # --- Python 函数定义 ---
27
 
28
- # Python 函数: 上传备份
 
 
29
  upload_backup() {
30
- file_path="$1"
31
- file_name="$2"
32
- token="$HF_TOKEN"
33
- repo_id="$DATASET_ID"
34
 
35
  echo "Preparing to upload backup file: $file_path as $file_name to Dataset: $repo_id"
36
 
 
 
37
  python3 -c "
38
  from huggingface_hub import HfApi
39
- import sys
40
- import os
41
  print(f'HF_TOKEN is set: {os.environ.get(\"HF_TOKEN\") is not None}')
42
  print(f'DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}')
43
- def manage_backups(api, repo_id_val, max_files=5):
44
  print('Managing old backups...')
45
- files = api.list_repo_files(repo_id=repo_id_val, repo_type='dataset')
46
- backup_files = [f for f in files if f.startswith('$BACKUP_PREFIX') and f.endswith('.tar.gz')]
47
- backup_files.sort()
48
- if len(backup_files) >= max_files:
49
- print(f'Found {len(backup_files)} backup files, maximum allowed is {max_files}.')
50
- # Corrected logic: delete oldest files until count is max_files - 1
51
- files_to_delete = backup_files[:len(backup_files) - max_files + 1]
52
- for file_to_delete in files_to_delete:
53
- try:
54
- print(f'Deleting old backup: {file_to_delete}')
55
- api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id_val, repo_type='dataset')
56
- print(f'Successfully deleted: {file_to_delete}')
57
- except Exception as e:
58
- print(f'Error deleting {file_to_delete}: {str(e)}')
59
- else:
60
- print('Number of backup files is within the limit.')
 
 
 
61
  api = HfApi(token='$token')
62
  try:
63
- repo_id_val = os.environ.get('DATASET_ID') # 从环境变量中获取 repo_id
64
- if not repo_id_val:
65
- raise ValueError('DATASET_ID environment variable is not set.')
66
  print(f'Uploading file: $file_path to {repo_id_val} as $file_name')
67
- api.upload_file(
68
- path_or_fileobj='$file_path',
69
- path_in_repo='$file_name',
70
- repo_id=repo_id_val,
71
- repo_type='dataset'
72
- )
73
  print(f'Successfully uploaded $file_name')
74
  manage_backups(api, repo_id_val)
75
  except Exception as e:
76
  print(f'Error uploading file: {str(e)}')
77
- sys.exit(1) # Exit if upload fails
78
  "
 
 
79
  }
80
 
81
- # Python 函数: 下载最新备份
82
  download_latest_backup() {
83
- token="$HF_TOKEN"
84
- repo_id="$DATASET_ID"
 
85
 
86
- echo "Preparing to download the latest backup from Dataset: $repo_id"
 
87
 
88
- python3 -c "
89
- from huggingface_hub import HfApi, hf_hub_download
90
- import sys
91
- import os
92
- import tarfile
93
- import tempfile
94
- import shutil
95
- import subprocess
96
 
 
 
 
 
 
97
  print(f'HF_TOKEN is set: {os.environ.get(\"HF_TOKEN\") is not None}')
98
  print(f'DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}')
 
99
 
100
  api = HfApi(token='$token')
101
  try:
102
- repo_id_val = os.environ.get('DATASET_ID') # 从环境变量中获取 repo_id
103
- if not repo_id_val:
104
- raise ValueError('DATASET_ID environment variable is not set.')
105
 
106
  print(f'Listing files in Dataset: {repo_id_val}')
107
  files = api.list_repo_files(repo_id=repo_id_val, repo_type='dataset')
108
- backup_files = [f for f in files if f.startswith('$BACKUP_PREFIX') and f.endswith('.tar.gz')]
109
 
110
  if not backup_files:
111
  print('No backup files found in the Dataset. Skipping restore.')
112
- # IMPORTANT: Exit with 0 to indicate success (no backup found is not a failure)
113
- sys.exit(0)
114
 
115
- latest_backup = sorted(backup_files)[-1]
116
  print(f'Latest backup file found: {latest_backup}')
117
 
118
  with tempfile.TemporaryDirectory() as temp_dir:
119
  print(f'Downloading {latest_backup} to temporary directory {temp_dir}...')
120
  try:
121
- filepath = hf_hub_download(
122
- repo_id=repo_id_val,
123
- filename=latest_backup,
124
- repo_type='dataset',
125
- local_dir=temp_dir,
126
- token=os.environ.get('HF_TOKEN') # Pass token explicitly if needed
127
- )
128
  except Exception as download_error:
129
- print(f'Error during hf_hub_download: {download_error}')
130
- # Attempt to list files again for debugging
131
- try:
132
- print('Attempting to list repo files again for debugging...')
133
- files_debug = api.list_repo_files(repo_id=repo_id_val, repo_type='dataset')
134
- print(f'Files found (debug): {files_debug}')
135
- except Exception as list_error:
136
- print(f'Error listing files during debug: {list_error}')
137
- sys.exit(1) # Exit if download fails
138
-
139
 
140
  if filepath and os.path.exists(filepath):
141
  print(f'Successfully downloaded backup to temporary directory: {filepath}')
142
-
143
- # Files/Dirs to restore (relative paths within CLOUDREVE_DIR)
144
  items_to_restore = ['cloudreve', 'cloudreve.db', 'config.ini']
145
-
146
- # Ensure target directory exists
147
  os.makedirs(\"$CLOUDREVE_DIR\", exist_ok=True)
 
148
 
149
- print('Listing contents before restore:')
150
- subprocess.run(['ls', '-lA', \"$CLOUDREVE_DIR\"], check=False) # Use -A to show hidden files
151
-
152
- # --- Safer Restore Logic ---
153
- # 1. Extract backup to a temporary location first
154
- extract_temp_dir = os.path.join(temp_dir, 'extracted_backup')
155
- os.makedirs(extract_temp_dir, exist_ok=True)
156
  print(f'Extracting backup archive: {filepath} to {extract_temp_dir}')
157
  try:
158
- # Use the 'r:gz' mode explicitly
159
- with tarfile.open(filepath, 'r:gz') as tar:
160
- tar.extractall(extract_temp_dir)
161
  print('Extraction complete.')
162
- except tarfile.ReadError as tar_err:
163
- print(f'Error reading tar file: {tar_err}')
164
- sys.exit(1)
165
- except Exception as extract_err:
166
- print(f'Error during extraction: {extract_err}')
167
- sys.exit(1)
168
-
169
 
170
- # 2. Check if essential files exist in the extracted backup
171
  essential_files_present = True
172
- print(f'Checking for essential items in extracted backup at {extract_temp_dir}:')
173
  for item in items_to_restore:
174
- extracted_item_path = os.path.join(extract_temp_dir, item)
175
- print(f'Checking for: {extracted_item_path}')
176
- if not os.path.exists(extracted_item_path):
177
- print(f'Error: Essential item "{item}" not found in extracted backup at {extracted_item_path}. Aborting restore.')
178
- essential_files_present = False
179
- # break # Stop checking (optional, maybe list all missing?)
180
-
181
  if not essential_files_present:
182
- print('Listing contents of extraction directory for debugging:')
183
- subprocess.run(['ls', '-lA', extract_temp_dir], check=False)
184
  sys.exit(1) # Abort if essential files are missing
185
 
186
- # 3. Delete existing items in the target directory
187
  print(f'Deleting existing items in $CLOUDREVE_DIR before restoring...')
188
  for item in items_to_restore:
189
  target_path = os.path.join(\"$CLOUDREVE_DIR\", item)
190
  if os.path.exists(target_path):
191
  try:
192
- if os.path.isdir(target_path) and not os.path.islink(target_path):
193
- print(f'Deleting directory: {target_path}')
194
- shutil.rmtree(target_path)
195
- else:
196
- print(f'Deleting file/link: {target_path}')
197
- os.remove(target_path)
198
- except OSError as e:
199
- print(f'Error deleting {target_path}: {e}. Continuing...')
200
-
201
 
202
- # 4. Move extracted items to the target directory
203
  print(f'Moving extracted items from {extract_temp_dir} to $CLOUDREVE_DIR...')
204
  for item in items_to_restore:
205
  source_path = os.path.join(extract_temp_dir, item)
206
- # Check if source exists before moving (should exist due to check above, but belt-and-suspenders)
207
  if os.path.exists(source_path):
208
  target_path = os.path.join(\"$CLOUDREVE_DIR\", item)
209
- try:
210
- print(f'Moving {source_path} to {target_path}')
211
- shutil.move(source_path, target_path)
212
- except Exception as move_err:
213
- print(f'Error moving {item}: {move_err}')
214
- # Decide if this is critical, maybe exit? For now, print and continue.
215
- else:
216
- # This case should ideally not happen if the check passed
217
- print(f'Warning: Source item {item} disappeared before move.')
218
-
219
 
220
  print(f'Successfully restored backup from {latest_backup}')
221
- print('Listing contents after restore:')
222
- subprocess.run(['ls', '-lA', \"$CLOUDREVE_DIR\"], check=False) # Use -A
 
 
 
223
  else:
224
  print(f'Error: Downloaded file path "{filepath}" does not exist or download failed.')
225
- sys.exit(1) # Exit if download path invalid
226
 
227
- except ValueError as ve:
228
- print(f'Configuration Error: {ve}')
229
- sys.exit(1)
230
- except Exception as e:
231
- print(f'Error during backup download/restore: {str(e)}')
232
- # Print traceback for more details
233
- import traceback
234
- traceback.print_exc()
235
- sys.exit(1) # Exit on error
236
  "
 
 
237
  }
238
 
239
-
240
- # --- Sync Function ---
241
- sync_data() {
242
- # **新增:等待 Cloudreve 启动完成**
243
- echo "Background Sync Process: Initializing..."
244
- echo "Waiting for Cloudreve service to become available on port 5212..."
245
- # 循环检查端口 5212 是否在监听,nc -z 返回 0 表示成功
246
- while ! nc -z 127.0.0.1 5212; do
247
- echo "Cloudreve not ready yet (port 5212 not open), sleeping for 5 seconds..."
248
- sleep 5
249
- done
250
- echo "Cloudreve service detected on port 5212. Starting main sync loop."
251
- # **等待逻辑结束**
252
-
253
- # --- 原有的 sync_data 循环逻辑开始 ---
254
- # echo "Background Sync Process Started" # 可以移除此行,上面的初始化消息更清晰
255
  while true; do
256
-
257
- # 确保基本文件存在(作为额外的保险,但理论上此时应该存在)
258
- if [ ! -f "$CONFIG_FILE_PATH" ] || [ ! -f "$DB_FILE_PATH" ] || [ ! -f "$EXECUTABLE_PATH" ]; then
259
- echo "WARN: Essential Cloudreve files missing even after port check. Waiting..."
260
- sleep 15
261
- continue # 跳过本次循环
262
- fi
263
-
264
- echo "Starting sync cycle at $(date)"
265
-
266
- # Define backup path and name
267
- timestamp=$(date +%Y%m%d_%H%M%S)
268
- backup_file="${BACKUP_PREFIX}_${timestamp}.tar.gz"
269
- backup_path="/tmp/${backup_file}" # Use /tmp for temporary files
270
-
271
- echo "Compressing Cloudreve data (executable, db, config) to: $backup_path"
272
- # Use -C to change directory, ensuring archive paths are relative
273
- # Only include the executable, db, and config file
274
- tar -czf "$backup_path" -C "$CLOUDREVE_DIR" \
275
- $(basename "$EXECUTABLE_PATH") \
276
- $(basename "$DB_FILE_PATH") \
277
- $(basename "$CONFIG_FILE_PATH")
278
-
279
- # Check if compression was successful (file exists and is not empty)
280
- if [ -s "$backup_path" ]; then
281
- echo "Compression complete. File size: $(ls -lh "$backup_path" | awk '{print $5}')"
282
- echo "Uploading backup to HuggingFace..."
283
- upload_backup "$backup_path" "${backup_file}"
284
- # Check exit status of upload_backup? The python script should exit non-zero on failure.
285
- if [ $? -ne 0 ]; then
286
- echo "Backup upload failed. Keeping local archive: $backup_path"
287
- else
288
- echo "Upload successful. Removing local archive."
289
- rm -f "$backup_path"
290
- fi
291
- else
292
- echo "Compression failed or created an empty file. Skipping upload."
293
- rm -f "$backup_path" # Remove potentially empty/corrupt file
294
- fi
295
-
296
- # Define sync interval (use environment variable or default to 3600 seconds = 1 hour)
297
- # **重要:确保这里的间隔设置合理,避免过于频繁导致资源超限**
298
- SYNC_INTERVAL=${SYNC_INTERVAL:-3600} # 默认改为 1 小时
299
- echo "Next sync in ${SYNC_INTERVAL} seconds..."
300
- sleep $SYNC_INTERVAL
301
  done
302
  }
303
 
304
- # --- Main Execution ---
305
 
306
- # 1. Attempt to restore from the latest backup on startup
307
- echo "Attempting to restore latest backup from HuggingFace..."
308
- download_latest_backup # <<< 调用函数发生在这里
309
- # Check the exit code of download_latest_backup
310
- # Python script exits with 0 if no backup found, 1 on error.
311
- ret_code=$?
312
- if [ $ret_code -ne 0 ]; then
313
- echo "CRITICAL: Backup restoration failed (Exit code: $ret_code). Exiting."
 
 
 
 
314
  exit 1
315
- elif [ $ret_code -eq 0 ]; then
316
- echo "Backup restore process finished (may have skipped if no backup found)."
 
 
 
 
317
  fi
 
 
318
 
319
 
320
- # 2. Check if config file exists after potential restore. If not, Cloudreve needs to run once to create it.
 
321
  if [ ! -f "$CONFIG_FILE_PATH" ]; then
322
- echo "Config file ($CONFIG_FILE_PATH) not found. Running Cloudreve once to generate initial config."
323
- # 首次运行时,不需要后台运行,让它生成配置就退出
324
  /opt/cloudreve/cloudreve -c "$CONFIG_FILE_PATH"
325
- # Cloudreve 在没有数据库时会初始化并打印密码然后退出
326
  if [ ! -f "$CONFIG_FILE_PATH" ]; then
327
  echo "CRITICAL: Cloudreve failed to create initial config file. Exiting."
328
  exit 1
329
  else
330
- echo "Initial config file created by first run. Please check logs for admin credentials if needed."
331
- # 确保数据库文件也可能已创建(即使是空的)
332
- if [ ! -f "$DB_FILE_PATH" ]; then
333
- echo "WARN: Database file might not have been created on first run, proceeding anyway."
334
- fi
335
  fi
336
  fi
337
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
338
 
339
- # 3. Start the background sync process
340
- # **注意:这一步仍然在 exec Cloudreve 之前**
341
- echo "Starting background data sync process (will wait for Cloudreve service internally)..."
342
- sync_data & # Run sync_data function in the background
343
- sync_pid=$! # Get PID of background sync process
344
-
345
- # 4. Start Cloudreve in the foreground using exec
346
- # 'exec' replaces the current shell process with the Cloudreve process.
347
- # This makes Cloudreve the main process of the container.
348
- echo "Starting Cloudreve application as the main process..."
349
- exec /opt/cloudreve/cloudreve -c "$CONFIG_FILE_PATH"
350
-
351
- # If exec fails, the script continues here.
352
- exec_failed_code=$?
353
- echo "CRITICAL: Failed to execute Cloudreve. Exit code: $exec_failed_code"
354
- # Attempt to kill the background sync process if exec failed
355
- kill $sync_pid 2>/dev/null
356
- exit $exec_failed_code
 
 
1
  #!/bin/bash
2
 
3
+ # Function to check if Cloudreve is running by checking the port
4
+ wait_for_cloudreve() {
5
+ echo "Waiting for Cloudreve service to become available on port 5212..."
6
+ # Loop until nc successfully connects (-z checks without sending data)
7
+ while ! nc -z 127.0.0.1 5212; do
8
+ echo "Cloudreve not ready yet (port 5212 not open), sleeping for 5 seconds..."
9
+ sleep 5
10
+ done
11
+ echo "Cloudreve service detected on port 5212."
12
+ }
13
+
14
+ # Function to perform a single backup action
15
+ perform_backup_once() {
16
+ echo "Performing single backup operation..."
17
+ local config_path="$1"
18
+ local db_path="$2"
19
+ local exe_path="$3"
20
+ local cloudreve_dir="$4"
21
+ local backup_prefix="$5"
22
+
23
+ if [ ! -f "$config_path" ] || [ ! -f "$db_path" ] || [ ! -f "$exe_path" ]; then
24
+ echo "WARN: Essential files missing for backup. Skipping this backup."
25
+ return 1
26
+ fi
27
+
28
+ local timestamp=$(date +%Y%m%d_%H%M%S)
29
+ local backup_file="${backup_prefix}_${timestamp}.tar.gz"
30
+ local backup_path="/tmp/${backup_file}" # Use /tmp for temporary files
31
+
32
+ echo "Compressing Cloudreve data (executable, db, config) to: $backup_path"
33
+ tar -czf "$backup_path" -C "$cloudreve_dir" \
34
+ $(basename "$exe_path") \
35
+ $(basename "$db_path") \
36
+ $(basename "$config_path")
37
+
38
+ if [ -s "$backup_path" ]; then
39
+ echo "Compression complete. File size: $(ls -lh "$backup_path" | awk '{print $5}')"
40
+ echo "Uploading backup to HuggingFace..."
41
+ # Call the previously defined upload_backup function
42
+ upload_backup "$backup_path" "${backup_file}"
43
+ local upload_status=$?
44
+ if [ $upload_status -ne 0 ]; then
45
+ echo "Backup upload failed. Keeping local archive: $backup_path"
46
+ return 1
47
+ else
48
+ echo "Upload successful. Removing local archive."
49
+ rm -f "$backup_path"
50
+ return 0
51
+ fi
52
+ else
53
+ echo "Compression failed or created an empty file. Skipping upload."
54
+ rm -f "$backup_path" # Remove potentially empty/corrupt file
55
+ return 1
56
+ fi
57
+ }
58
+
59
+
60
+ # --- Start Script Execution ---
61
+
62
+ # Check Hugging Face Token and Dataset ID environment variables
63
  if [[ -z "$HF_TOKEN" ]] || [[ -z "$DATASET_ID" ]]; then
64
+ echo "Starting without backup functionality - missing HF_TOKEN or DATASET_ID"
65
+ # Ensure Aria2 starts if needed, even without backup/restore
66
+ # Example (adjust options as needed from your Dockerfile CMD):
67
+ # aria2c --daemon=true [OTHER_OPTIONS] &
 
 
68
  exec /opt/cloudreve/cloudreve -c /opt/cloudreve/config.ini
69
+ exit 0
70
  fi
71
 
72
+ # Activate Python virtual environment
73
  echo "Activating Python venv..."
74
  source /opt/venv/bin/activate
75
 
76
+ # Define Cloudreve paths and backup prefix
77
  CLOUDREVE_DIR="/opt/cloudreve"
78
  BACKUP_PREFIX="cloudreve_backup"
79
  CONFIG_FILE_PATH="/opt/cloudreve/config.ini"
80
  DB_FILE_PATH="/opt/cloudreve/cloudreve.db"
81
  EXECUTABLE_PATH="/opt/cloudreve/cloudreve"
82
 
 
83
 
84
+ # --- Python Function Definitions (Ensure these are complete) ---
85
+
86
+ # Python Function: Upload Backup
87
  upload_backup() {
88
+ local file_path="$1"
89
+ local file_name="$2"
90
+ local token="$HF_TOKEN"
91
+ local repo_id="$DATASET_ID"
92
 
93
  echo "Preparing to upload backup file: $file_path as $file_name to Dataset: $repo_id"
94
 
95
+ # Use python3 -c "..." with the full Python code block here
96
+ # (Using placeholder for brevity, replace with your actual Python code)
97
  python3 -c "
98
  from huggingface_hub import HfApi
99
+ import sys, os
 
100
  print(f'HF_TOKEN is set: {os.environ.get(\"HF_TOKEN\") is not None}')
101
  print(f'DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}')
102
+ def manage_backups(api, repo_id_val, max_files=50): # Increased max_files
103
  print('Managing old backups...')
104
+ try:
105
+ files = api.list_repo_files(repo_id=repo_id_val, repo_type='dataset')
106
+ backup_files = sorted([f for f in files if f.startswith('$BACKUP_PREFIX') and f.endswith('.tar.gz')])
107
+ if len(backup_files) >= max_files:
108
+ print(f'Found {len(backup_files)} backup files, maximum allowed is {max_files}.')
109
+ files_to_delete = backup_files[:len(backup_files) - max_files + 1]
110
+ print(f'Will delete {len(files_to_delete)} old backups.')
111
+ for file_to_delete in files_to_delete:
112
+ try:
113
+ print(f'Deleting old backup: {file_to_delete}')
114
+ api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id_val, repo_type='dataset', token=os.environ.get('HF_TOKEN'))
115
+ print(f'Successfully deleted: {file_to_delete}')
116
+ except Exception as e:
117
+ print(f'Error deleting {file_to_delete}: {str(e)}')
118
+ else:
119
+ print(f'Number of backup files ({len(backup_files)}) is within the limit ({max_files}).')
120
+ except Exception as e:
121
+ print(f'Error listing or managing backups: {e}')
122
+
123
  api = HfApi(token='$token')
124
  try:
125
+ repo_id_val = os.environ.get('DATASET_ID')
126
+ if not repo_id_val: raise ValueError('DATASET_ID environment variable is not set.')
 
127
  print(f'Uploading file: $file_path to {repo_id_val} as $file_name')
128
+ api.upload_file(path_or_fileobj='$file_path', path_in_repo='$file_name', repo_id=repo_id_val, repo_type='dataset', token=os.environ.get('HF_TOKEN'))
 
 
 
 
 
129
  print(f'Successfully uploaded $file_name')
130
  manage_backups(api, repo_id_val)
131
  except Exception as e:
132
  print(f'Error uploading file: {str(e)}')
133
+ sys.exit(1) # Indicate failure
134
  "
135
+ # Capture python exit code
136
+ return $?
137
  }
138
 
139
+ # Python Function: Download Latest Backup
140
  download_latest_backup() {
141
+ local token="$HF_TOKEN"
142
+ local repo_id="$DATASET_ID"
143
+ local download_flag_file="/tmp/backup_restored.flag" # Flag file
144
 
145
+ # Remove any previous flag file
146
+ rm -f "$download_flag_file"
147
 
148
+ echo "Preparing to download the latest backup from Dataset: $repo_id"
 
 
 
 
 
 
 
149
 
150
+ # Use python3 -c "..." with the full Python code block here
151
+ # (Using placeholder for brevity, replace with your actual Python code)
152
+ python3 -c "
153
+ from huggingface_hub import HfApi, hf_hub_download
154
+ import sys, os, tarfile, tempfile, shutil, subprocess
155
  print(f'HF_TOKEN is set: {os.environ.get(\"HF_TOKEN\") is not None}')
156
  print(f'DATASET_ID is set: {os.environ.get(\"DATASET_ID\") is not None}')
157
+ flag_file = '$download_flag_file' # Get flag file path
158
 
159
  api = HfApi(token='$token')
160
  try:
161
+ repo_id_val = os.environ.get('DATASET_ID')
162
+ if not repo_id_val: raise ValueError('DATASET_ID environment variable is not set.')
 
163
 
164
  print(f'Listing files in Dataset: {repo_id_val}')
165
  files = api.list_repo_files(repo_id=repo_id_val, repo_type='dataset')
166
+ backup_files = sorted([f for f in files if f.startswith('$BACKUP_PREFIX') and f.endswith('.tar.gz')])
167
 
168
  if not backup_files:
169
  print('No backup files found in the Dataset. Skipping restore.')
170
+ sys.exit(0) # Success: No backup found is not an error for the script flow
 
171
 
172
+ latest_backup = backup_files[-1]
173
  print(f'Latest backup file found: {latest_backup}')
174
 
175
  with tempfile.TemporaryDirectory() as temp_dir:
176
  print(f'Downloading {latest_backup} to temporary directory {temp_dir}...')
177
  try:
178
+ filepath = hf_hub_download(repo_id=repo_id_val, filename=latest_backup, repo_type='dataset', local_dir=temp_dir, token=os.environ.get('HF_TOKEN'))
 
 
 
 
 
 
179
  except Exception as download_error:
180
+ print(f'Error during hf_hub_download: {download_error}'); sys.exit(1)
 
 
 
 
 
 
 
 
 
181
 
182
  if filepath and os.path.exists(filepath):
183
  print(f'Successfully downloaded backup to temporary directory: {filepath}')
 
 
184
  items_to_restore = ['cloudreve', 'cloudreve.db', 'config.ini']
 
 
185
  os.makedirs(\"$CLOUDREVE_DIR\", exist_ok=True)
186
+ print('Listing contents before restore:'); subprocess.run(['ls', '-lA', \"$CLOUDREVE_DIR\"], check=False)
187
 
188
+ extract_temp_dir = os.path.join(temp_dir, 'extracted_backup'); os.makedirs(extract_temp_dir, exist_ok=True)
 
 
 
 
 
 
189
  print(f'Extracting backup archive: {filepath} to {extract_temp_dir}')
190
  try:
191
+ with tarfile.open(filepath, 'r:gz') as tar: tar.extractall(extract_temp_dir)
 
 
192
  print('Extraction complete.')
193
+ except Exception as extract_err: print(f'Error during extraction: {extract_err}'); sys.exit(1)
 
 
 
 
 
 
194
 
 
195
  essential_files_present = True
 
196
  for item in items_to_restore:
197
+ extracted_item_path = os.path.join(extract_temp_dir, item)
198
+ if not os.path.exists(extracted_item_path):
199
+ print(f'Error: Essential item "{item}" not found in extracted backup at {extracted_item_path}. Aborting restore.')
200
+ essential_files_present = False; # break # Keep checking all missing items
 
 
 
201
  if not essential_files_present:
202
+ print('Listing contents of extraction directory for debugging:'); subprocess.run(['ls', '-lA', extract_temp_dir], check=False)
 
203
  sys.exit(1) # Abort if essential files are missing
204
 
 
205
  print(f'Deleting existing items in $CLOUDREVE_DIR before restoring...')
206
  for item in items_to_restore:
207
  target_path = os.path.join(\"$CLOUDREVE_DIR\", item)
208
  if os.path.exists(target_path):
209
  try:
210
+ if os.path.isdir(target_path) and not os.path.islink(target_path): shutil.rmtree(target_path)
211
+ else: os.remove(target_path)
212
+ except OSError as e: print(f'Error deleting {target_path}: {e}. Continuing...')
 
 
 
 
 
 
213
 
 
214
  print(f'Moving extracted items from {extract_temp_dir} to $CLOUDREVE_DIR...')
215
  for item in items_to_restore:
216
  source_path = os.path.join(extract_temp_dir, item)
 
217
  if os.path.exists(source_path):
218
  target_path = os.path.join(\"$CLOUDREVE_DIR\", item)
219
+ try: shutil.move(source_path, target_path)
220
+ except Exception as move_err: print(f'Error moving {item}: {move_err}') # Continue for now
221
+ else: print(f'Warning: Source item {item} disappeared before move.')
 
 
 
 
 
 
 
222
 
223
  print(f'Successfully restored backup from {latest_backup}')
224
+ print('Listing contents after restore:'); subprocess.run(['ls', '-lA', \"$CLOUDREVE_DIR\"], check=False)
225
+ # Create the flag file only on successful restore
226
+ with open(flag_file, 'w') as f: f.write('restored')
227
+ print(f'Created restore flag file: {flag_file}')
228
+ sys.exit(0) # Indicate successful restore completion
229
  else:
230
  print(f'Error: Downloaded file path "{filepath}" does not exist or download failed.')
231
+ sys.exit(1)
232
 
233
+ except ValueError as ve: print(f'Configuration Error: {ve}'); sys.exit(1)
234
+ except Exception as e: import traceback; print(f'Error during backup download/restore: {str(e)}'); traceback.print_exc(); sys.exit(1)
 
 
 
 
 
 
 
235
  "
236
+ # Capture python exit code
237
+ return $?
238
  }
239
 
240
+ # --- Periodic Sync Function (Loop only) ---
241
+ sync_data_loop() {
242
+ echo "Starting Periodic Background Sync Loop..."
 
 
 
 
 
 
 
 
 
 
 
 
 
243
  while true; do
244
+ # Wait for the defined interval
245
+ local sync_interval=${SYNC_INTERVAL:-3600} # Default to 1 hour
246
+ echo "Periodic Sync: Next check in ${sync_interval} seconds..."
247
+ sleep $sync_interval
248
+
249
+ # Perform the backup
250
+ echo "Periodic Sync: Starting backup cycle at $(date)"
251
+ perform_backup_once "$CONFIG_FILE_PATH" "$DB_FILE_PATH" "$EXECUTABLE_PATH" "$CLOUDREVE_DIR" "$BACKUP_PREFIX"
252
+ # We don't necessarily need to exit the loop if one backup fails
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
253
  done
254
  }
255
 
 
256
 
257
+ # --- Main Execution Logic ---
258
+
259
+ # 1. Attempt Initial Restore
260
+ echo "--- Step 1: Attempting Initial Restore ---"
261
+ download_latest_backup
262
+ restore_exit_code=$?
263
+ restore_flag_file="/tmp/backup_restored.flag"
264
+
265
+ # Check restore outcome
266
+ backup_was_restored=false
267
+ if [ $restore_exit_code -ne 0 ]; then
268
+ echo "CRITICAL: Backup restoration attempt failed with error (Exit code: $restore_exit_code). Exiting."
269
  exit 1
270
+ elif [ -f "$restore_flag_file" ]; then
271
+ echo "Restore successful: Backup found and restored."
272
+ backup_was_restored=true
273
+ else
274
+ echo "Restore skipped: No backup file found in the dataset."
275
+ backup_was_restored=false
276
  fi
277
+ # Clean up flag file regardless
278
+ rm -f "$restore_flag_file"
279
 
280
 
281
+ # 2. Initial Cloudreve Run (If necessary)
282
+ echo "--- Step 2: Checking for Initial Config ---"
283
  if [ ! -f "$CONFIG_FILE_PATH" ]; then
284
+ echo "Config file ($CONFIG_FILE_PATH) not found. Running Cloudreve once to generate initial config..."
 
285
  /opt/cloudreve/cloudreve -c "$CONFIG_FILE_PATH"
 
286
  if [ ! -f "$CONFIG_FILE_PATH" ]; then
287
  echo "CRITICAL: Cloudreve failed to create initial config file. Exiting."
288
  exit 1
289
  else
290
+ echo "Initial config file created by first run. Check logs for credentials if needed."
 
 
 
 
291
  fi
292
  fi
293
 
294
+ # 3. Start Cloudreve Main Process (in background temporarily to allow checks)
295
+ echo "--- Step 3: Starting Cloudreve Main Process (background initially) ---"
296
+ /opt/cloudreve/cloudreve -c "$CONFIG_FILE_PATH" &
297
+ cloudreve_pid=$!
298
+
299
+ # 4. Wait for Cloudreve to be Ready
300
+ echo "--- Step 4: Waiting for Cloudreve Service ---"
301
+ wait_for_cloudreve # Use the function defined earlier
302
+
303
+ # 5. Perform Initial Backup (If No Restore Occurred)
304
+ echo "--- Step 5: Checking for Initial Backup ---"
305
+ if [ "$backup_was_restored" = false ]; then
306
+ echo "No backup was restored, performing initial backup of current state..."
307
+ perform_backup_once "$CONFIG_FILE_PATH" "$DB_FILE_PATH" "$EXECUTABLE_PATH" "$CLOUDREVE_DIR" "$BACKUP_PREFIX"
308
+ if [ $? -ne 0 ]; then
309
+ echo "Warning: Initial backup failed."
310
+ # Decide whether to continue or exit if initial backup fails
311
+ else
312
+ echo "Initial backup completed successfully."
313
+ fi
314
+ else
315
+ echo "Backup was restored, skipping initial backup."
316
+ fi
317
 
318
+ # 6. Start Periodic Sync Loop in Background
319
+ echo "--- Step 6: Starting Periodic Sync Loop ---"
320
+ sync_data_loop &
321
+ sync_loop_pid=$!
322
+
323
+ # 7. Bring Cloudreve to Foreground / Wait for it
324
+ # Since Cloudreve was started in the background, we now wait for it.
325
+ # If Cloudreve exits, this script will also exit.
326
+ echo "--- Step 7: Cloudreve is running. Monitoring PID: $cloudreve_pid ---"
327
+ wait $cloudreve_pid
328
+ exit_code=$?
329
+ echo "Cloudreve process exited with code $exit_code."
330
+
331
+ # Optional: attempt to clean up background sync loop if Cloudreve exits unexpectedly
332
+ echo "Attempting to terminate background sync loop (PID: $sync_loop_pid)..."
333
+ kill $sync_loop_pid 2>/dev/null
334
+ wait $sync_loop_pid 2>/dev/null # Wait briefly for it to terminate
335
+
336
+ exit $exit_code