tkgray commited on
Commit
88b3855
·
verified ·
1 Parent(s): ca5eedc

Update sync_data.sh

Browse files
Files changed (1) hide show
  1. sync_data.sh +20 -20
sync_data.sh CHANGED
@@ -1,12 +1,12 @@
1
  #!/bin/sh
2
 
3
- # 检查环境变量
4
  if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
5
- echo "缺少必要的环境变量 HF_TOKEN DATASET_ID"
6
  exit 1
7
  fi
8
 
9
- # 上传备份
10
  cat > /tmp/hf_sync.py << 'EOL'
11
  from huggingface_hub import HfApi
12
  import sys
@@ -22,9 +22,9 @@ def manage_backups(api, repo_id, max_files=50):
22
  for file_to_delete in files_to_delete:
23
  try:
24
  api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type="dataset")
25
- print(f'已删除旧备份: {file_to_delete}')
26
  except Exception as e:
27
- print(f'删除 {file_to_delete} 时出错: {str(e)}')
28
 
29
  def upload_backup(file_path, file_name, token, repo_id):
30
  api = HfApi(token=token)
@@ -35,13 +35,13 @@ def upload_backup(file_path, file_name, token, repo_id):
35
  repo_id=repo_id,
36
  repo_type="dataset"
37
  )
38
- print(f"成功上传 {file_name}")
39
 
40
  manage_backups(api, repo_id)
41
  except Exception as e:
42
- print(f"文件上传出错: {str(e)}")
43
 
44
- # 下载最新备份
45
  def download_latest_backup(token, repo_id):
46
  try:
47
  api = HfApi(token=token)
@@ -49,7 +49,7 @@ def download_latest_backup(token, repo_id):
49
  backup_files = [f for f in files if f.startswith('webui_backup_') and f.endswith('.db')]
50
 
51
  if not backup_files:
52
- print("未找到备份文件")
53
  return
54
 
55
  latest_backup = sorted(backup_files)[-1]
@@ -63,10 +63,10 @@ def download_latest_backup(token, repo_id):
63
  if filepath and os.path.exists(filepath):
64
  os.makedirs('./data', exist_ok=True)
65
  os.system(f'cp "{filepath}" ./data/webui.db')
66
- print(f"成功从 {latest_backup} 恢复备份")
67
 
68
  except Exception as e:
69
- print(f"下载备份时出错: {str(e)}")
70
 
71
  if __name__ == "__main__":
72
  action = sys.argv[1]
@@ -81,35 +81,35 @@ if __name__ == "__main__":
81
  download_latest_backup(token, repo_id)
82
  EOL
83
 
84
- # 首次启动时下载最新备份
85
- echo "正在从 HuggingFace 下载最新备份..."
86
  python3 /tmp/hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}"
87
 
88
- # 同步函数
89
  sync_data() {
90
  while true; do
91
- echo "开始同步进程 $(date)"
92
 
93
  if [ -f "./data/webui.db" ]; then
94
  timestamp=$(date +%Y%m%d_%H%M%S)
95
  backup_file="webui_backup_${timestamp}.db"
96
 
97
- # 复制数据库文件
98
  cp ./data/webui.db "/tmp/${backup_file}"
99
 
100
- echo "正在上传备份到 HuggingFace..."
101
  python3 /tmp/hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
102
 
103
  rm -f "/tmp/${backup_file}"
104
  else
105
- echo "数据库文件不存在,等待下次同步..."
106
  fi
107
 
108
  SYNC_INTERVAL=${SYNC_INTERVAL:-7200}
109
- echo "下次同步将在 ${SYNC_INTERVAL} 秒后进行..."
110
  sleep $SYNC_INTERVAL
111
  done
112
  }
113
 
114
- # 后台启动同步进程
115
  sync_data &
 
1
  #!/bin/sh
2
 
3
+ # Check environment variables
4
  if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then
5
+ echo "Missing required environment variables HF_TOKEN or DATASET_ID"
6
  exit 1
7
  fi
8
 
9
+ # Upload backup
10
  cat > /tmp/hf_sync.py << 'EOL'
11
  from huggingface_hub import HfApi
12
  import sys
 
22
  for file_to_delete in files_to_delete:
23
  try:
24
  api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type="dataset")
25
+ print(f'Deleted old backup: {file_to_delete}')
26
  except Exception as e:
27
+ print(f'Error deleting {file_to_delete}: {str(e)}')
28
 
29
  def upload_backup(file_path, file_name, token, repo_id):
30
  api = HfApi(token=token)
 
35
  repo_id=repo_id,
36
  repo_type="dataset"
37
  )
38
+ print(f"Successfully uploaded {file_name}")
39
 
40
  manage_backups(api, repo_id)
41
  except Exception as e:
42
+ print(f"File upload error: {str(e)}")
43
 
44
+ # Download latest backup
45
  def download_latest_backup(token, repo_id):
46
  try:
47
  api = HfApi(token=token)
 
49
  backup_files = [f for f in files if f.startswith('webui_backup_') and f.endswith('.db')]
50
 
51
  if not backup_files:
52
+ print("Backup file not found")
53
  return
54
 
55
  latest_backup = sorted(backup_files)[-1]
 
63
  if filepath and os.path.exists(filepath):
64
  os.makedirs('./data', exist_ok=True)
65
  os.system(f'cp "{filepath}" ./data/webui.db')
66
+ print(f"Successfully restored backup from {latest_backup}")
67
 
68
  except Exception as e:
69
+ print(f"Error downloading backup: {str(e)}")
70
 
71
  if __name__ == "__main__":
72
  action = sys.argv[1]
 
81
  download_latest_backup(token, repo_id)
82
  EOL
83
 
84
+ # Download latest backup on first startup
85
+ echo "Downloading latest backup from HuggingFace..."
86
  python3 /tmp/hf_sync.py download "${HF_TOKEN}" "${DATASET_ID}"
87
 
88
+ # Sync function
89
  sync_data() {
90
  while true; do
91
+ echo "Starting sync process $(date)"
92
 
93
  if [ -f "./data/webui.db" ]; then
94
  timestamp=$(date +%Y%m%d_%H%M%S)
95
  backup_file="webui_backup_${timestamp}.db"
96
 
97
+ # Copy database file
98
  cp ./data/webui.db "/tmp/${backup_file}"
99
 
100
+ echo "Uploading backup to HuggingFace..."
101
  python3 /tmp/hf_sync.py upload "${HF_TOKEN}" "${DATASET_ID}" "/tmp/${backup_file}" "${backup_file}"
102
 
103
  rm -f "/tmp/${backup_file}"
104
  else
105
+ echo "Database file does not exist, waiting for next sync..."
106
  fi
107
 
108
  SYNC_INTERVAL=${SYNC_INTERVAL:-7200}
109
+ echo "Next sync will be in ${SYNC_INTERVAL} seconds..."
110
  sleep $SYNC_INTERVAL
111
  done
112
  }
113
 
114
+ # Start sync process in background
115
  sync_data &