chawin.chen
commited on
Commit
·
5f103ce
1
Parent(s):
fae1594
init
Browse files
utils.py
CHANGED
|
@@ -4,6 +4,7 @@ import os
|
|
| 4 |
import re
|
| 5 |
import shutil
|
| 6 |
import threading
|
|
|
|
| 7 |
from typing import Optional
|
| 8 |
|
| 9 |
import cv2
|
|
@@ -253,11 +254,10 @@ def ensure_bos_resources(force_download: bool = False) -> bool:
|
|
| 253 |
_BOS_DOWNLOAD_COMPLETED = True
|
| 254 |
return True
|
| 255 |
|
| 256 |
-
|
| 257 |
for target in targets:
|
| 258 |
if not isinstance(target, dict):
|
| 259 |
logger.warning("无效的 BOS 下载配置项: %r", target)
|
| 260 |
-
results.append(False)
|
| 261 |
continue
|
| 262 |
|
| 263 |
prefix = target.get("bos_prefix")
|
|
@@ -266,16 +266,57 @@ def ensure_bos_resources(force_download: bool = False) -> bool:
|
|
| 266 |
|
| 267 |
if not prefix or not destination:
|
| 268 |
logger.warning("缺少必要字段,无法处理 BOS 下载配置: %r", target)
|
| 269 |
-
results.append(False)
|
| 270 |
continue
|
| 271 |
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
|
| 280 |
all_ready = all(results) if results else True
|
| 281 |
if all_ready:
|
|
|
|
| 4 |
import re
|
| 5 |
import shutil
|
| 6 |
import threading
|
| 7 |
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 8 |
from typing import Optional
|
| 9 |
|
| 10 |
import cv2
|
|
|
|
| 254 |
_BOS_DOWNLOAD_COMPLETED = True
|
| 255 |
return True
|
| 256 |
|
| 257 |
+
download_jobs = []
|
| 258 |
for target in targets:
|
| 259 |
if not isinstance(target, dict):
|
| 260 |
logger.warning("无效的 BOS 下载配置项: %r", target)
|
|
|
|
| 261 |
continue
|
| 262 |
|
| 263 |
prefix = target.get("bos_prefix")
|
|
|
|
| 266 |
|
| 267 |
if not prefix or not destination:
|
| 268 |
logger.warning("缺少必要字段,无法处理 BOS 下载配置: %r", target)
|
|
|
|
| 269 |
continue
|
| 270 |
|
| 271 |
+
download_jobs.append(
|
| 272 |
+
{
|
| 273 |
+
"description": description,
|
| 274 |
+
"prefix": prefix,
|
| 275 |
+
"destination": destination,
|
| 276 |
+
}
|
| 277 |
+
)
|
| 278 |
+
|
| 279 |
+
if not download_jobs:
|
| 280 |
+
logger.info("未找到有效的 BOS 下载目标,跳过资源同步")
|
| 281 |
+
_BOS_DOWNLOAD_COMPLETED = True
|
| 282 |
+
return True
|
| 283 |
+
|
| 284 |
+
results = []
|
| 285 |
+
max_workers = min(len(download_jobs), max(os.cpu_count() or 1, 1))
|
| 286 |
+
if max_workers <= 0:
|
| 287 |
+
max_workers = 1
|
| 288 |
+
|
| 289 |
+
with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="bos-sync") as executor:
|
| 290 |
+
future_to_job = {}
|
| 291 |
+
for job in download_jobs:
|
| 292 |
+
logger.info(
|
| 293 |
+
"准备同步 BOS 资源: %s (prefix=%s -> %s)",
|
| 294 |
+
job["description"],
|
| 295 |
+
job["prefix"],
|
| 296 |
+
job["destination"],
|
| 297 |
+
)
|
| 298 |
+
future = executor.submit(
|
| 299 |
+
download_bos_directory,
|
| 300 |
+
job["prefix"],
|
| 301 |
+
job["destination"],
|
| 302 |
+
force_download=force_download,
|
| 303 |
+
)
|
| 304 |
+
future_to_job[future] = job
|
| 305 |
+
|
| 306 |
+
for future in as_completed(future_to_job):
|
| 307 |
+
job = future_to_job[future]
|
| 308 |
+
description = job["description"]
|
| 309 |
+
try:
|
| 310 |
+
success = future.result()
|
| 311 |
+
except Exception as exc:
|
| 312 |
+
logger.warning("BOS 资源同步异常: %s (%s)", description, exc)
|
| 313 |
+
success = False
|
| 314 |
+
|
| 315 |
+
if success:
|
| 316 |
+
logger.info("BOS 资源已就绪: %s", description)
|
| 317 |
+
else:
|
| 318 |
+
logger.warning("BOS 资源同步失败: %s", description)
|
| 319 |
+
results.append(success)
|
| 320 |
|
| 321 |
all_ready = all(results) if results else True
|
| 322 |
if all_ready:
|