Update sync_data.sh

sync_data.sh

@@ -13,6 +13,39 @@ source /opt/venv/bin/activate
 # 定义 Cloudreve 主程序目录
 CLOUDREVE_DIR="/opt/cloudreve"
 BACKUP_PREFIX="cloudreve_backup"
+HF_DATA_DIR="/opt/cloudreve/hf_uploaded" # 用于记录已上传到 HF 的文件
+
+# 创建 HF 上传记录目录
+mkdir -p "$HF_DATA_DIR"
+
+# Python 函数: 上传文件到 Hugging Face Dataset
+upload_file_to_dataset() {
+    local_file_path="$1"
+    relative_path=$(echo "$local_file_path" | sed "s|$CLOUDREVE_DIR/data/||")
+    token="$HF_TOKEN"
+    repo_id="$DATASET_ID"
+
+    echo "Preparing to upload file: $local_file_path to Dataset: $repo_id at path: $relative_path"
+
+    python3 -c "
+from huggingface_hub import HfApi
+import os
+
+api = HfApi(token='$token')
+try:
+    repo_id_val = os.environ.get('DATASET_ID')
+    print(f'Uploading file: '$local_file_path' to {repo_id_val} as '$relative_path'')
+    api.upload_file(
+        path_or_fileobj='$local_file_path',
+        path_in_repo='$relative_path',
+        repo_id=repo_id_val,
+        repo_type='dataset'
+    )
+    print(f'Successfully uploaded '$relative_path'')
+except Exception as e:
+    print(f'Error uploading file: {str(e)}')
+"
+}
 
 # Python 函数: 上传备份
 upload_backup() {
@@ -109,41 +142,25 @@ try:
 
         if filepath and os.path.exists(filepath):
             print(f'Successfully downloaded backup to temporary directory: {filepath}')
-            print(\"Before restoring backup:\")
-            import subprocess
-            subprocess.run(['ls', '-l', \"$CLOUDREVE_DIR\"], shell=True, check=False)
+            echo \"Before restoring backup:\"
+            ls -l \"$CLOUDREVE_DIR\"
 
             # 删除现有的 Cloudreve 目录和配置文件
-            import shutil
-            cloudreve_path = os.path.join(\"$CLOUDREVE_DIR\", \"cloudreve\")
-            cloudreve_db_path = os.path.join(\"$CLOUDREVE_DIR\", \"cloudreve.db\")
-            config_ini_path = os.path.join(\"$CLOUDREVE_DIR\", \"config.ini\")
-            data_path = os.path.join(\"$CLOUDREVE_DIR\", \"data\")
-
-            if os.path.exists(cloudreve_path):
-                print(f'Deleting: {cloudreve_path}')
-                shutil.rmtree(cloudreve_path, ignore_errors=True)
-            if os.path.exists(cloudreve_db_path):
-                print(f'Deleting: {cloudreve_db_path}')
-                os.remove(cloudreve_db_path)
-            if os.path.exists(config_ini_path):
-                print(f'Deleting: {config_ini_path}')
-                os.remove(config_ini_path)
-            if os.path.exists(data_path):
-                print(f'Deleting: {data_path}')
-                shutil.rmtree(data_path, ignore_errors=True)
-            print(\"Deletion complete.\")
-
-            print(f'Extracting backup archive: {filepath} to $CLOUDREVE_DIR')
-            import tarfile
-            with tarfile.open(filepath, 'r:gz') as tar:
-                tar.extractall(\"$CLOUDREVE_DIR\")
-            print(f'Successfully restored backup from {latest_backup}')
-
-            print(\"After restoring backup:\")
-            subprocess.run(['ls', '-l', \"$CLOUDREVE_DIR\"], shell=True, check=False)
+            rm -rf \"$CLOUDREVE_DIR/cloudreve\"
+            rm -f \"$CLOUDREVE_DIR/cloudreve.db\"
+            rm -f \"$CLOUDREVE_DIR/config.ini\"
+            rm -rf \"$CLOUDREVE_DIR/data\" # 也删除数据目录
+
+            echo \"Deletion complete.\"
+
+            echo \"Extracting backup archive: $filepath to $CLOUDREVE_DIR\"
+            tar -xzf \"$filepath\" -C \"$CLOUDREVE_DIR\"
+            echo \"Successfully restored backup from $latest_backup\"
+
+            echo \"After restoring backup:\"
+            ls -l \"$CLOUDREVE_DIR\"
         else:
-            print('Error during file download.')
+            echo 'Error during file download.'
 
 except Exception as e:
     print(f'Error downloading backup: {str(e)}')
@@ -156,10 +173,21 @@ download_latest_backup
 
 # 同步函数
 sync_data() {
-    echo "SYNC_DATA FUNCTION IS RUNNING" # 添加了这一行
+    echo "SYNC_DATA FUNCTION IS RUNNING"
     while true; do
         echo "Starting sync process at $(date)"
 
+        # 检查 Cloudreve data 目录并上传新文件
+        if [ -d "$CLOUDREVE_DIR/data" ]; then
+            find "$CLOUDREVE_DIR/data" -type f -print0 | while IFS= read -r -d $'\0' file; do
+                if [ ! -f "$HF_DATA_DIR/$(basename "$file")" ]; then
+                    echo "New file found: $file"
+                    upload_file_to_dataset "$file"
+                    touch "$HF_DATA_DIR/$(basename "$file")" # 创建已上传标记
+                fi
+            done
+        fi
+
         if [ -d "$CLOUDREVE_DIR" ]; then
             echo "Before compression:"
             ls -l \"$CLOUDREVE_DIR\"
@@ -183,14 +211,14 @@ sync_data() {
             echo "Cloudreve directory does not exist yet, waiting for next sync..."
         fi
 
-        SYNC_INTERVAL=${SYNC_INTERVAL:-60} # 默认同步间隔改为 60 秒
+        SYNC_INTERVAL=${SYNC_INTERVAL:-300} # 默认同步间隔改为 5 分钟
         echo "Next sync in ${SYNC_INTERVAL} seconds..."
         sleep $SYNC_INTERVAL
     done
 }
 
 # 延迟启动同步脚本，给 Cloudreve 一些启动时间
-sleep 10
+sleep 30
 
 # 后台启动同步进程
 sync_data &