import os import sys import subprocess sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from utils.logger import setup_logger from utils.json import field_getter log = setup_logger(__name__) def run_step(cmd: list[str], allow_fail: bool = False): log.info("==> %s", " ".join(cmd)) try: subprocess.run(cmd, check=True) except subprocess.CalledProcessError as e: if allow_fail: log.warning(f"step失敗を無視します: {cmd} (code={e.returncode})") else: raise def main(): files = field_getter("config/files.json") # Step 0: Download embeddings run_step([sys.executable, "scripts/0_download_data.py"], allow_fail=True) # Step 1: Sudachi user dict (optional but recommended before tokenization) run_step([sys.executable, "scripts/1_build_dict.py"], allow_fail=True) # Step 2: circles.json run_step([sys.executable, "scripts/2_create_circles_data.py"]) # Step 3: synonyms cache run_step([sys.executable, "scripts/3_build_synonyms_from_sudachi.py"]) # idempotent # Step 4: bm25 meta run_step([sys.executable, "scripts/4_prepare_bm25f_meta.py"]) # needs projects.json # Step 5: tf_token run_step([sys.executable, "scripts/5_prepare_tf_token.py"]) # needs projects.json # Step 6: embeddings (.vec がある場合のみ) vec_path = files("embeddings.fasttext_vec") if os.path.exists(vec_path): run_step( [sys.executable, "scripts/6_build_word_embeddings.py"] ) # needs tf_token, bm25_meta else: log.info(".vec が見つからないため Step 6 をスキップします: %s", vec_path) # Step 7: circle names run_step( [sys.executable, "scripts/7_prepare_circle_names.py"] ) # needs projects.json log.info("全ステップ完了") if __name__ == "__main__": main()