ktsn-ud commited on
Commit
f4ede90
·
1 Parent(s): 59424a5

cc.ja.300.binをFastAPI起動時にダウンロードするコードを追加

Browse files
Files changed (1) hide show
  1. api/main.py +31 -1
api/main.py CHANGED
@@ -11,6 +11,7 @@ from pydantic import BaseModel
11
  from dotenv import load_dotenv
12
 
13
  from utils.logger import setup_logger
 
14
  from api.search.engine import SearchEngine
15
  import schemas.projects as schema_projects
16
 
@@ -32,12 +33,42 @@ async def get_api_key(key: str = Security(api_key_header)):
32
 
33
 
34
  # --- App ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
35
  @asynccontextmanager
36
  async def lifespan(app: FastAPI):
37
  """アプリケーションの起動時と終了時に実行されるコード"""
38
  # --- 起動時処理 ---
39
  log.info("Initializing search engine and loading assets...")
40
  load_dotenv()
 
 
 
 
 
41
  engine.initialize()
42
  log.info("Initialization complete.")
43
  yield
@@ -46,7 +77,6 @@ async def lifespan(app: FastAPI):
46
  log.info("Shutting down search engine...")
47
 
48
 
49
- engine = SearchEngine()
50
  app = FastAPI(lifespan=lifespan)
51
 
52
 
 
11
  from dotenv import load_dotenv
12
 
13
  from utils.logger import setup_logger
14
+ from utils.json import get_file_path_from_config
15
  from api.search.engine import SearchEngine
16
  import schemas.projects as schema_projects
17
 
 
33
 
34
 
35
  # --- App ---
36
+ engine = SearchEngine()
37
+
38
+
39
+ def download_embeddings() -> None:
40
+ """Hugging Face Datasets から Embeddings をダウンロードする"""
41
+ from huggingface_hub import hf_hub_download
42
+
43
+ # フォルダがなかったら新規作成
44
+ target_dir = "resources/embeddings"
45
+ os.makedirs(target_dir, exist_ok=True)
46
+
47
+ # ダウンロード
48
+ path = hf_hub_download(
49
+ repo_id=os.getenv("HF_EMBEDDINGS_REPO_ID"),
50
+ repo_type="dataset",
51
+ filename="cc.ja.300.bin",
52
+ local_dir=target_dir,
53
+ local_dir_use_symlinks=False,
54
+ token=os.getenv("HF_TOKEN"),
55
+ )
56
+
57
+ log.info(f"Embeddings downloaded: {path}")
58
+ return
59
+
60
+
61
  @asynccontextmanager
62
  async def lifespan(app: FastAPI):
63
  """アプリケーションの起動時と終了時に実行されるコード"""
64
  # --- 起動時処理 ---
65
  log.info("Initializing search engine and loading assets...")
66
  load_dotenv()
67
+ if os.path.exists(get_file_path_from_config("embeddings.fasttext_bin")):
68
+ log.info("FastText embeddings found locally.")
69
+ else:
70
+ log.info("FastText embeddings not found locally. Downloading...")
71
+ download_embeddings()
72
  engine.initialize()
73
  log.info("Initialization complete.")
74
  yield
 
77
  log.info("Shutting down search engine...")
78
 
79
 
 
80
  app = FastAPI(lifespan=lifespan)
81
 
82