init

Dockerfile

@@ -1,12 +1,12 @@
 FROM python:3.10-slim
 
 ENV TZ=Asia/Shanghai \
-    OUTPUT_DIR=/opt/output \
-    IMAGES_DIR=/opt/images \
-    MODELS_PATH=/opt/models \
-    DEEPFACE_HOME=/opt/models \
-    FAISS_INDEX_DIR=/opt/faiss \
-    CELEBRITY_SOURCE_DIR=/opt/chinese_celeb_dataset \
+    OUTPUT_DIR=/opt/data/output \
+    IMAGES_DIR=/opt/data/images \
+    MODELS_PATH=/opt/data/models \
+    DEEPFACE_HOME=/opt/data/models \
+    FAISS_INDEX_DIR=/opt/data/faiss \
+    CELEBRITY_SOURCE_DIR=/opt/data/chinese_celeb_dataset \
     GENDER_CONFIDENCE=1 \
     UPSCALE_SIZE=2 \
     AGE_CONFIDENCE=0.1 \
@@ -27,8 +27,8 @@ ENV TZ=Asia/Shanghai \
     ENABLE_ANIME_PRELOAD=false \
     ENABLE_LOGGING=true \
     BEAUTY_ADJUST_ENABLED=true \
-    RVM_LOCAL_REPO=/app/RobustVideoMatting \
-    RVM_WEIGHTS_PATH=/opt/models/torch/hub/checkpoints/rvm_resnet50.pth \
+    RVM_LOCAL_REPO=/opt/data/RobustVideoMatting \
+    RVM_WEIGHTS_PATH=/opt/data/models/torch/hub/checkpoints/rvm_resnet50.pth \
     RVM_MODEL=resnet50 \
     AUTO_INIT_GFPGAN=false \
     AUTO_INIT_DDCOLOR=false \
@@ -43,7 +43,7 @@ ENV TZ=Asia/Shanghai \
     FEMALE_AGE_ADJUSTMENT=4 \
     HOSTNAME=HG
 
-RUN mkdir -p  /opt/chinese_celeb_dataset /opt/faiss /opt/models /opt/images /opt/output
+RUN mkdir -p  /opt/data/chinese_celeb_dataset /opt/data/faiss /opt/data/models /opt/data/images /opt/data/output
 WORKDIR /app
 COPY requirements.txt .
 COPY *.py /app/

app.py

@@ -17,8 +17,20 @@ from config import (
     ENABLE_LOGGING,
 )
 from database import close_mysql_pool, init_mysql_pool
+from utils import ensure_bos_resources
 
 logger.info("Starting to import api_routes module...")
+
+try:
+    t_bos_start = time.perf_counter()
+    if not ensure_bos_resources():
+        raise RuntimeError("无法从 BOS 同步模型与数据，请检查凭证与网络")
+    bos_time = time.perf_counter() - t_bos_start
+    logger.info(f"BOS resources synchronized successfully, time: {bos_time:.3f}s")
+except Exception as exc:
+    logger.error(f"BOS resource preparation failed: {exc}")
+    raise
+
 try:
     t_start = time.perf_counter()
     from api_routes import api_router

config.py

@@ -176,15 +176,24 @@ try:
 except (ImportError, AttributeError) as e:
     print(f"Warning: PyTorch/PyArrow compatibility patch failed: {e}")
     pass
-IMAGES_DIR = os.environ.get("IMAGES_DIR", "~/app/data/images")
+IMAGES_DIR = os.environ.get("IMAGES_DIR", "/opt/data/images")
 OUTPUT_DIR = IMAGES_DIR
 
 # 明星图库目录配置
 CELEBRITY_SOURCE_DIR = os.environ.get(
-    "CELEBRITY_SOURCE_DIR", "~/apps/chinese_celeb_imgs"
+    "CELEBRITY_SOURCE_DIR", "/opt/data/chinese_celeb_dataset"
 ).strip()
 if CELEBRITY_SOURCE_DIR:
-    CELEBRITY_SOURCE_DIR = os.path.expanduser(CELEBRITY_SOURCE_DIR)
+    CELEBRITY_SOURCE_DIR = os.path.abspath(os.path.expanduser(CELEBRITY_SOURCE_DIR))
+
+CELEBRITY_DATASET_DIR = os.path.abspath(
+    os.path.expanduser(
+        os.environ.get(
+            "CELEBRITY_DATASET_DIR",
+            CELEBRITY_SOURCE_DIR or "/opt/data/chinese_celeb_dataset",
+        )
+    )
+)
 
 CELEBRITY_FIND_THRESHOLD = float(
     os.environ.get("CELEBRITY_FIND_THRESHOLD", 0.88)
@@ -202,6 +211,10 @@ BOS_ENDPOINT = os.environ.get(
 ).strip()
 BOS_BUCKET_NAME = os.environ.get("BOS_BUCKET_NAME", "hbgs-travel").strip()
 BOS_IMAGE_DIR = os.environ.get("BOS_IMAGE_DIR", "20220808").strip()
+BOS_MODELS_PREFIX = os.environ.get("BOS_MODELS_PREFIX", "20220620/models").strip()
+BOS_CELEBRITY_PREFIX = os.environ.get(
+    "BOS_CELEBRITY_PREFIX", "20220620/chinese_celeb_dataset"
+).strip()
 _bos_enabled_env = os.environ.get("BOS_UPLOAD_ENABLED")
 if _bos_enabled_env is not None:
     BOS_UPLOAD_ENABLED = _bos_enabled_env.lower() in ("1", "true", "on")
@@ -216,12 +229,17 @@ else:
     )
 APP_SECRET_TOKEN = os.environ.get("APP_SECRET_TOKEN", "Abdc@q1")
 HOSTNAME = os.environ.get("HOSTNAME", "default-hostname")
-MODELS_PATH = os.environ.get("MODELS_PATH", "~/apps/ai/models")
+MODELS_PATH = os.path.abspath(
+    os.path.expanduser(os.environ.get("MODELS_PATH", "/opt/data/models"))
+)
+MODELS_DOWNLOAD_DIR = os.path.abspath(
+    os.path.expanduser(os.environ.get("MODELS_DOWNLOAD_DIR", MODELS_PATH))
+)
 DEEPFACE_HOME = os.environ.get("DEEPFACE_HOME", "~/apps/ai")
 os.environ["DEEPFACE_HOME"] = DEEPFACE_HOME
 
 # 设置GFPGAN相关模型下载路径
-GFPGAN_MODEL_DIR = "~/apps/ai/models"
+GFPGAN_MODEL_DIR = MODELS_DOWNLOAD_DIR
 os.makedirs(GFPGAN_MODEL_DIR, exist_ok=True)
 
 # 设置各种模型库的下载目录环境变量
@@ -286,6 +304,20 @@ AUTO_INIT_RVM = os.environ.get("AUTO_INIT_RVM", "false").lower() in ("1", "true"
 CLEANUP_INTERVAL_HOURS = float(os.environ.get("CLEANUP_INTERVAL_HOURS", 12.0))  # 清理任务执行间隔（小时），默认1小时
 CLEANUP_AGE_HOURS = float(os.environ.get("CLEANUP_AGE_HOURS", 12.0))  # 清理文件的年龄阈值（小时），默认1小时
 
+# BOS 自动同步清单：定义 BOS 路径和本地目录的映射，启动时可迭代该结构完成批量下载
+BOS_DOWNLOAD_TARGETS = [
+    {
+        "description": "明星图库数据集",
+        "bos_prefix": BOS_CELEBRITY_PREFIX,
+        "destination": CELEBRITY_DATASET_DIR,
+    },
+    {
+        "description": "AI 模型权重",
+        "bos_prefix": BOS_MODELS_PREFIX,
+        "destination": MODELS_DOWNLOAD_DIR,
+    },
+]
+
 log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
 log_level = getattr(logging, log_level_str, logging.INFO)
 

start_local.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+export TZ=Asia/Shanghai
+export HOSTNAME=HG
+uvicorn app:app --workers 1 --loop asyncio --http httptools --host 0.0.0.0 --port 7860 --timeout-keep-alive 600
+

utils.py

@@ -4,6 +4,7 @@ import os
 import re
 import shutil
 import threading
+from typing import Optional
 
 import cv2
 import numpy as np
@@ -27,11 +28,14 @@ from config import (
     BOS_BUCKET_NAME,
     BOS_IMAGE_DIR,
     BOS_UPLOAD_ENABLED,
+    BOS_DOWNLOAD_TARGETS,
 )
 
 _BOS_CLIENT = None
 _BOS_CLIENT_INITIALIZED = False
 _BOS_CLIENT_LOCK = threading.Lock()
+_BOS_DOWNLOAD_LOCK = threading.Lock()
+_BOS_DOWNLOAD_COMPLETED = False
 _IMAGES_DIR_ABS = os.path.abspath(os.path.expanduser(IMAGES_DIR))
 
 
@@ -109,6 +113,177 @@ def _get_bos_client():
     return _BOS_CLIENT
 
 
+def _normalize_bos_prefix(prefix: Optional[str]) -> str:
+    value = (prefix or "").strip()
+    if not value:
+        return ""
+    value = value.strip("/")
+    if not value:
+        return ""
+    return f"{value}/" if not value.endswith("/") else value
+
+
+def _directory_has_files(path: str) -> bool:
+    try:
+        for _root, _dirs, files in os.walk(path):
+            if files:
+                return True
+    except Exception:
+        return False
+    return False
+
+
+def download_bos_directory(prefix: str, destination_dir: str, *, force_download: bool = False) -> bool:
+    """
+    将 BOS 上的指定前缀目录同步到本地。
+    :param prefix: BOS 对象前缀，例如 'models/' 或 '20220620/models'
+    :param destination_dir: 本地目标目录
+    :param force_download: 是否强制重新下载（忽略本地已存在的文件）
+    :return: 是否确保目录可用
+    """
+    client = _get_bos_client()
+    if client is None:
+        logger.warning("BOS 客户端不可用，无法下载资源（prefix=%s）", prefix)
+        return False
+
+    dest_dir = os.path.abspath(os.path.expanduser(destination_dir))
+    try:
+        os.makedirs(dest_dir, exist_ok=True)
+    except Exception as exc:
+        logger.error("创建本地目录失败: %s (%s)", dest_dir, exc)
+        return False
+
+    normalized_prefix = _normalize_bos_prefix(prefix)
+
+    # 未强制下载且目录已有文件时直接跳过，避免重复下载
+    if not force_download and _directory_has_files(dest_dir):
+        logger.info("本地目录已存在文件，跳过下载: %s -> %s", normalized_prefix or "<root>", dest_dir)
+        return True
+
+    paginate_kwargs = {"Bucket": BOS_BUCKET_NAME}
+    if normalized_prefix:
+        paginate_kwargs["Prefix"] = normalized_prefix if normalized_prefix.endswith("/") else f"{normalized_prefix}/"
+
+    found_any = False
+    downloaded = 0
+    skipped = 0
+
+    try:
+        paginator = client.get_paginator("list_objects_v2")
+        for page in paginator.paginate(**paginate_kwargs):
+            for obj in page.get("Contents", []):
+                key = obj.get("Key")
+                if not key:
+                    continue
+                if normalized_prefix:
+                    prefix_with_slash = normalized_prefix if normalized_prefix.endswith("/") else f"{normalized_prefix}/"
+                    if not key.startswith(prefix_with_slash):
+                        continue
+                    relative_key = key[len(prefix_with_slash):]
+                else:
+                    relative_key = key
+
+                if not relative_key or relative_key.endswith("/"):
+                    continue
+                found_any = True
+
+                target_path = os.path.join(dest_dir, relative_key)
+                target_dir = os.path.dirname(target_path)
+                os.makedirs(target_dir, exist_ok=True)
+
+                expected_size = obj.get("Size")
+                if (
+                    not force_download
+                    and os.path.exists(target_path)
+                    and expected_size is not None
+                    and expected_size == os.path.getsize(target_path)
+                ):
+                    skipped += 1
+                    logger.info("文件已存在且大小一致，跳过下载: %s", relative_key)
+                    continue
+
+                tmp_path = f"{target_path}.download"
+                try:
+                    size_mb = (expected_size or 0) / (1024 * 1024)
+                    logger.info("开始下载: %s (%.2f MB)", relative_key, size_mb)
+                    client.download_file(Bucket=BOS_BUCKET_NAME, Key=key, Filename=tmp_path)
+                    os.replace(tmp_path, target_path)
+                    downloaded += 1
+                    logger.info("下载完成: %s", relative_key)
+                except Exception as exc:
+                    logger.warning("下载失败: %s (%s)", key, exc)
+                    try:
+                        if os.path.exists(tmp_path):
+                            os.remove(tmp_path)
+                    except Exception:
+                        pass
+    except Exception as exc:
+        logger.warning("遍历 BOS 目录失败: %s", exc)
+        return False
+
+    if not found_any:
+        logger.warning("在 BOS 桶 %s 中未找到前缀 '%s' 的内容", BOS_BUCKET_NAME, normalized_prefix or "<root>")
+        return False
+
+    logger.info(
+        "BOS 同步完成 prefix=%s -> %s 下载=%d 跳过=%d",
+        normalized_prefix or "<root>",
+        dest_dir,
+        downloaded,
+        skipped,
+    )
+    return downloaded > 0 or skipped > 0
+
+
+def ensure_bos_resources(force_download: bool = False) -> bool:
+    """
+    根据配置的 BOS_DOWNLOAD_TARGETS 同步启动所需的模型与数据资源。
+    :param force_download: 是否强制重新同步所有资源
+    :return: 资源是否已准备就绪
+    """
+    global _BOS_DOWNLOAD_COMPLETED
+
+    with _BOS_DOWNLOAD_LOCK:
+        if _BOS_DOWNLOAD_COMPLETED and not force_download:
+            return True
+
+        targets = BOS_DOWNLOAD_TARGETS or []
+        if not targets:
+            logger.info("未配置 BOS 下载目标，跳过资源同步")
+            _BOS_DOWNLOAD_COMPLETED = True
+            return True
+
+        results = []
+        for target in targets:
+            if not isinstance(target, dict):
+                logger.warning("无效的 BOS 下载配置项: %r", target)
+                results.append(False)
+                continue
+
+            prefix = target.get("bos_prefix")
+            destination = target.get("destination")
+            description = target.get("description") or prefix or "<unnamed>"
+
+            if not prefix or not destination:
+                logger.warning("缺少必要字段，无法处理 BOS 下载配置: %r", target)
+                results.append(False)
+                continue
+
+            logger.info("准备同步 BOS 资源: %s (prefix=%s -> %s)", description, prefix, destination)
+            success = download_bos_directory(prefix, destination, force_download=force_download)
+            if success:
+                logger.info("BOS 资源已就绪: %s", description)
+            else:
+                logger.warning("BOS 资源同步失败: %s", description)
+            results.append(success)
+
+        all_ready = all(results) if results else True
+        if all_ready:
+            _BOS_DOWNLOAD_COMPLETED = True
+
+        return all_ready
+
+
 def upload_file_to_bos(file_path: str, object_name: str | None = None) -> bool:
     """
     将指定文件上传到 BOS，失败不会抛出异常。