chawin.chen commited on
Commit
5f103ce
·
1 Parent(s): fae1594
Files changed (1) hide show
  1. utils.py +51 -10
utils.py CHANGED
@@ -4,6 +4,7 @@ import os
4
  import re
5
  import shutil
6
  import threading
 
7
  from typing import Optional
8
 
9
  import cv2
@@ -253,11 +254,10 @@ def ensure_bos_resources(force_download: bool = False) -> bool:
253
  _BOS_DOWNLOAD_COMPLETED = True
254
  return True
255
 
256
- results = []
257
  for target in targets:
258
  if not isinstance(target, dict):
259
  logger.warning("无效的 BOS 下载配置项: %r", target)
260
- results.append(False)
261
  continue
262
 
263
  prefix = target.get("bos_prefix")
@@ -266,16 +266,57 @@ def ensure_bos_resources(force_download: bool = False) -> bool:
266
 
267
  if not prefix or not destination:
268
  logger.warning("缺少必要字段,无法处理 BOS 下载配置: %r", target)
269
- results.append(False)
270
  continue
271
 
272
- logger.info("准备同步 BOS 资源: %s (prefix=%s -> %s)", description, prefix, destination)
273
- success = download_bos_directory(prefix, destination, force_download=force_download)
274
- if success:
275
- logger.info("BOS 资源已就绪: %s", description)
276
- else:
277
- logger.warning("BOS 资源同步失败: %s", description)
278
- results.append(success)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
279
 
280
  all_ready = all(results) if results else True
281
  if all_ready:
 
4
  import re
5
  import shutil
6
  import threading
7
+ from concurrent.futures import ThreadPoolExecutor, as_completed
8
  from typing import Optional
9
 
10
  import cv2
 
254
  _BOS_DOWNLOAD_COMPLETED = True
255
  return True
256
 
257
+ download_jobs = []
258
  for target in targets:
259
  if not isinstance(target, dict):
260
  logger.warning("无效的 BOS 下载配置项: %r", target)
 
261
  continue
262
 
263
  prefix = target.get("bos_prefix")
 
266
 
267
  if not prefix or not destination:
268
  logger.warning("缺少必要字段,无法处理 BOS 下载配置: %r", target)
 
269
  continue
270
 
271
+ download_jobs.append(
272
+ {
273
+ "description": description,
274
+ "prefix": prefix,
275
+ "destination": destination,
276
+ }
277
+ )
278
+
279
+ if not download_jobs:
280
+ logger.info("未找到有效的 BOS 下载目标,跳过资源同步")
281
+ _BOS_DOWNLOAD_COMPLETED = True
282
+ return True
283
+
284
+ results = []
285
+ max_workers = min(len(download_jobs), max(os.cpu_count() or 1, 1))
286
+ if max_workers <= 0:
287
+ max_workers = 1
288
+
289
+ with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="bos-sync") as executor:
290
+ future_to_job = {}
291
+ for job in download_jobs:
292
+ logger.info(
293
+ "准备同步 BOS 资源: %s (prefix=%s -> %s)",
294
+ job["description"],
295
+ job["prefix"],
296
+ job["destination"],
297
+ )
298
+ future = executor.submit(
299
+ download_bos_directory,
300
+ job["prefix"],
301
+ job["destination"],
302
+ force_download=force_download,
303
+ )
304
+ future_to_job[future] = job
305
+
306
+ for future in as_completed(future_to_job):
307
+ job = future_to_job[future]
308
+ description = job["description"]
309
+ try:
310
+ success = future.result()
311
+ except Exception as exc:
312
+ logger.warning("BOS 资源同步异常: %s (%s)", description, exc)
313
+ success = False
314
+
315
+ if success:
316
+ logger.info("BOS 资源已就绪: %s", description)
317
+ else:
318
+ logger.warning("BOS 资源同步失败: %s", description)
319
+ results.append(success)
320
 
321
  all_ready = all(results) if results else True
322
  if all_ready: