scripts/build_all.py

import os
import sys
import subprocess

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))

from utils.logger import setup_logger
from utils.json import field_getter

log = setup_logger(__name__)


def run_step(cmd: list[str], allow_fail: bool = False):
    log.info("==> %s", " ".join(cmd))
    try:
        subprocess.run(cmd, check=True)
    except subprocess.CalledProcessError as e:
        if allow_fail:
            log.warning(f"step失敗を無視します: {cmd} (code={e.returncode})")
        else:
            raise


def main():
    files = field_getter("config/files.json")

    # Step 0: Download embeddings
    run_step([sys.executable, "scripts/0_download_data.py"], allow_fail=True)

    # Step 1: Sudachi user dict (optional but recommended before tokenization)
    run_step([sys.executable, "scripts/1_build_dict.py"], allow_fail=True)

    # Step 2: circles.json
    run_step([sys.executable, "scripts/2_create_circles_data.py"])

    # Step 3: synonyms cache
    run_step([sys.executable, "scripts/3_build_synonyms_from_sudachi.py"])  # idempotent

    # Step 4: bm25 meta
    run_step([sys.executable, "scripts/4_prepare_bm25f_meta.py"])  # needs projects.json

    # Step 5: tf_token
    run_step([sys.executable, "scripts/5_prepare_tf_token.py"])  # needs projects.json

    # Step 6: embeddings (.vec がある場合のみ)
    vec_path = files("embeddings.fasttext_vec")
    if os.path.exists(vec_path):
        run_step(
            [sys.executable, "scripts/6_build_word_embeddings.py"]
        )  # needs tf_token, bm25_meta
    else:
        log.info(".vec が見つからないため Step 6 をスキップします: %s", vec_path)

    # Step 7: circle names
    run_step(
        [sys.executable, "scripts/7_prepare_circle_names.py"]
    )  # needs projects.json
    log.info("全ステップ完了")


if __name__ == "__main__":
    main()