インポートに関するエラー修正と、vecのダウンロードを追加

scripts/0_download_data.py

@@ -1,10 +1,16 @@
 import os
+import sys
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
 from utils.logger import setup_logger
 from utils.json import get_file_path_from_config
 
 log = setup_logger(__name__)
 
+fast_text_bin_path = get_file_path_from_config("embeddings.fasttext_bin")
+fast_text_vec_path = get_file_path_from_config("embeddings.fasttext_vec")
+
 
 def download_embeddings() -> None:
     """Hugging Face Datasets から Embeddings をダウンロードする"""
@@ -17,22 +23,34 @@ def download_embeddings() -> None:
     log.info("Downloading embeddings from Hugging Face Datasets...")
 
     # ダウンロード
-    path = hf_hub_download(
-        repo_id=os.getenv("HF_EMBEDDINGS_REPO_ID"),
-        repo_type="dataset",
-        filename="cc.ja.300.bin",
-        local_dir=target_dir,
-        local_dir_use_symlinks=False,
-        token=os.getenv("HF_TOKEN"),
-    )
-
-    log.info(f"Embeddings downloaded: {path}")
+    if os.path.exists(fast_text_bin_path):
+        log.info("Embeddings binary already exists. Skipping download.")
+    else:
+        path = hf_hub_download(
+            repo_id=os.getenv("HF_EMBEDDINGS_REPO_ID"),
+            repo_type="dataset",
+            filename="cc.ja.300.bin",
+            local_dir=target_dir,
+            local_dir_use_symlinks=False,
+            token=os.getenv("HF_TOKEN"),
+        )
+        log.info(f"Embeddings downloaded: {path}")
+
+    if os.path.exists(fast_text_vec_path):
+        log.info("Embeddings vector already exists. Skipping download.")
+    else:
+        path = hf_hub_download(
+            repo_id=os.getenv("HF_EMBEDDINGS_REPO_ID"),
+            repo_type="dataset",
+            filename="cc.ja.300.vec",
+            local_dir=target_dir,
+            local_dir_use_symlinks=False,
+            token=os.getenv("HF_TOKEN"),
+        )
+        log.info(f"Embeddings downloaded: {path}")
+
     return
 
 
 if __name__ == "__main__":
-    fast_text_bin_path = get_file_path_from_config("embeddings.fasttext_bin")
-    if os.path.exists(fast_text_bin_path):
-        log.info("Embeddings already exist. Skipping download.")
-    else:
-        download_embeddings()
+    download_embeddings()