codex生成: 4以降のscript全て

config/files.json

@@ -10,5 +10,16 @@
     "projects": {
         "original_csv": "resources/test_data.csv",
         "projects_json": "data/generated/projects.json"
+    },
+    "bm25": {
+        "bm25_meta": "data/generated/bm25_meta.json",
+        "tf_token": "data/generated/tf_token.json"
+    },
+    "embeddings": {
+        "fasttext_vec": "resources/embeddings/cc.ja.300.vec",
+        "fasttext_bin": "resources/embeddings/cc.ja.300.bin",
+        "word_vocab": "data/generated/word_vocab.json",
+        "word_vectors": "data/generated/word_vectors.npz",
+        "doc_vectors": "data/generated/doc_vectors.npy"
     }
-}
+}

config/search_model.json

@@ -1,30 +1,56 @@
 {
-    "target_pos_l1": [
-        "名詞", "動詞"
-    ],
-    "target_fields": [
-        "title",
-        "organization",
-        "description",
-        "prComment",
-        "prCommentLong",
-        "reading"
-    ],
-    "synonyms": {
-        "enable": true,
-        "sources": {
-            "sudachi": true,
-            "custom_json": "resources/synonyms_custom.json"
-        },
-        "limits": {
-            "max_expansions_per_term": 4,
-            "max_query_variants": 5,
-            "min_char_len": 2
-        },
-        "banlist": [
-            "部",
-            "会",
-            "サークル"
-        ]
+  "target_pos_l1": [
+    "名詞",
+    "動詞",
+    "形容詞",
+    "形容動詞語幹"
+  ],
+  "target_fields": [
+    "title",
+    "organization",
+    "description",
+    "prComment",
+    "prCommentLong",
+    "reading"
+  ],
+  "synonyms": {
+    "enable": true,
+    "sources": {
+      "sudachi": true,
+      "custom_json": "resources/synonyms_custom.json"
+    },
+    "limits": {
+      "max_expansions_per_term": 4,
+      "max_query_variants": 5,
+      "min_char_len": 2
+    },
+    "banlist": [
+      "部",
+      "会",
+      "サークル"
+    ]
+  },
+  "bm25f": {
+    "k1": 1.2,
+    "b": 0.75,
+    "field_weights": {
+      "title": 2.0,
+      "organization": 1.5,
+      "description": 1.0,
+      "prComment": 1.0,
+      "prCommentLong": 0.8,
+      "reading": 0.6
     }
-}
+  },
+  "word_sim": {
+    "enable": false,
+    "mode": "soft",
+    "alpha": 0.7
+  },
+  "query_subword": {
+    "enable": true,
+    "path": "resources/embeddings/cc.ja.300.bin",
+    "oov_weight": 0.8,
+    "cache_size": 50000
+  }
+}

docs/files.md

@@ -2,9 +2,16 @@
 - `cc.ja.300.bin`: OOV（Out of Vocabrary）時に使用 更新不要
 - `generated/`: 毎年更新
     - `projects.json`: API返却用データ
+        - `scripts/1_build_dict.py`で生成
     - `bm25_meta.json`: BM25Fメタデータ
+        - `scripts/4_prepare_bm25f_meta.py`で生成
     - `tf_token.json`: フィールド別TF/トークン
+        - `scripts/5_prepare_tf_token.py`で生成
     - `synonyms_cache.json`: Sudachi同義語キャッシュ
+        - `scripts/3_build_synonyms_from_sudachi.py`から生成
     - `word_vocab.json`
+        - `scripts/6_build_word_embeddings.py`で生成
     - `word_vectors.npz`
+        - `scripts/6_build_word_embeddings.py`で生成
     - `doc_vectors.npy`
+        - `scripts/6_build_word_embeddings.py`で生成

scripts/4_prepare_bm25f_meta.py

@@ -0,0 +1,117 @@
+import os
+import sys
+import json
+from collections import defaultdict
+from typing import Dict, List
+
+from sudachipy import dictionary, tokenizer
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from utils.logger import setup_logger
+from utils.json import get_file_path_from_config, field_getter, json_dumps
+
+log = setup_logger(__name__)
+
+
+def load_configs():
+    search = field_getter("config/search_model.json")
+    files = field_getter("config/files.json")
+
+    target_pos_l1: List[str] = search("target_pos_l1")
+    target_fields: List[str] = search("target_fields")
+    ban_list: List[str] = search("synonyms.banlist")
+
+    stopwords_path = files("sudachi.stopwords")
+    with open(stopwords_path, encoding="utf-8") as f:
+        stopwords = set(json.load(f))
+
+    sudachi_config_path = files("sudachi.sudachi_config")
+
+    return target_pos_l1, target_fields, set(ban_list), stopwords, sudachi_config_path
+
+
+def build_tokenizer(sudachi_config_path: str):
+    tok = dictionary.Dictionary(config_path=sudachi_config_path).create()
+    mode = tokenizer.Tokenizer.SplitMode.A
+    return tok, mode
+
+
+def tokenize(text: str, tok, mode, target_pos_l1: List[str], ban_list: set, stopwords: set) -> List[str]:
+    if not text:
+        return []
+    out: List[str] = []
+    for m in tok.tokenize(text, mode):
+        base = m.normalized_form().lower().strip()
+        if not base:
+            continue
+        pos = m.part_of_speech()
+        if pos[0] not in target_pos_l1:
+            continue
+        if base in stopwords or base in ban_list:
+            continue
+        out.append(base)
+    return out
+
+
+def main():
+    log.info("BM25Fメタデータ(bm25_meta.json)を生成します")
+    try:
+        target_pos_l1, target_fields, ban_list, stopwords, sudachi_config_path = load_configs()
+    except Exception as e:
+        log.error(f"設定の読み込みに失敗しました: {e}")
+        sys.exit(1)
+
+    projects_path = get_file_path_from_config("projects.projects_json", "data/generated/projects.json")
+    output_path = get_file_path_from_config("bm25.bm25_meta", "data/generated/bm25_meta.json")
+
+    try:
+        with open(projects_path, encoding="utf-8") as f:
+            projects = json.load(f)
+    except Exception as e:
+        log.error(f"projects.jsonの読み込みに失敗しました: {e}")
+        sys.exit(1)
+
+    tok, mode = build_tokenizer(sudachi_config_path)
+
+    N = len(projects)
+    df: Dict[str, int] = defaultdict(int)
+    field_token_lens_sum: Dict[str, int] = {f: 0 for f in target_fields}
+
+    log.info(f"ドキュメント数: {N}")
+
+    for p in projects:
+        seen_in_doc = set()
+        for field in target_fields:
+            text = p.get(field) or ""
+            toks = tokenize(str(text), tok, mode, target_pos_l1, ban_list, stopwords)
+            field_token_lens_sum[field] += len(toks)
+            for t in set(toks):
+                if t not in seen_in_doc:
+                    df[t] += 1
+                    seen_in_doc.add(t)
+
+    # IDF 計算（BM25で一般的な +0.5 smoothing と +1 オフセット）
+    idf: Dict[str, float] = {}
+    for term, dfi in df.items():
+        idf_val = max(0.0, ( ( (N - dfi + 0.5) / (dfi + 0.5) ) ))
+        # 数値安定化のためlog1p
+        import math
+
+        idf[term] = math.log1p(idf_val)
+
+    avg_len = {field: (field_token_lens_sum[field] / N if N > 0 else 0.0) for field in target_fields}
+
+    meta = {
+        "N": N,
+        "avg_len": avg_len,
+        "idf": idf,
+    }
+
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    json_dumps(meta, output_path)
+    log.info(f"bm25_meta.json を出力しました: {output_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/5_prepare_tf_token.py

@@ -0,0 +1,124 @@
+import os
+import sys
+import json
+from collections import Counter
+from typing import Dict, List
+
+from sudachipy import dictionary, tokenizer
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from utils.logger import setup_logger
+from utils.json import get_file_path_from_config, field_getter, json_dumps
+import schemas.tf_token as tf_schema
+
+log = setup_logger(__name__)
+
+
+def load_configs():
+    search = field_getter("config/search_model.json")
+    files = field_getter("config/files.json")
+
+    target_pos_l1: List[str] = search("target_pos_l1")
+    target_fields: List[str] = search("target_fields")
+    ban_list: List[str] = search("synonyms.banlist")
+
+    stopwords_path = files("sudachi.stopwords")
+    with open(stopwords_path, encoding="utf-8") as f:
+        stopwords = set(json.load(f))
+
+    sudachi_config_path = files("sudachi.sudachi_config")
+
+    return target_pos_l1, target_fields, set(ban_list), stopwords, sudachi_config_path
+
+
+def build_tokenizer(sudachi_config_path: str):
+    tok = dictionary.Dictionary(config_path=sudachi_config_path).create()
+    mode = tokenizer.Tokenizer.SplitMode.A
+    return tok, mode
+
+
+def tokenize(text: str, tok, mode, target_pos_l1: List[str], ban_list: set, stopwords: set) -> List[str]:
+    if not text:
+        return []
+    out: List[str] = []
+    for m in tok.tokenize(text, mode):
+        base = m.normalized_form().lower().strip()
+        if not base:
+            continue
+        pos = m.part_of_speech()
+        if pos[0] not in target_pos_l1:
+            continue
+        if base in stopwords or base in ban_list:
+            continue
+        out.append(base)
+    return out
+
+
+def main():
+    log.info("フィールド別TF/トークン(tf_token.json)を生成します")
+    try:
+        target_pos_l1, target_fields, ban_list, stopwords, sudachi_config_path = load_configs()
+    except Exception as e:
+        log.error(f"設定の読み込みに失敗しました: {e}")
+        sys.exit(1)
+
+    projects_path = get_file_path_from_config("projects.projects_json", "data/generated/projects.json")
+    output_path = get_file_path_from_config("bm25.tf_token", "data/generated/tf_token.json")
+
+    try:
+        with open(projects_path, encoding="utf-8") as f:
+            projects = json.load(f)
+    except Exception as e:
+        log.error(f"projects.jsonの読み込みに失敗しました: {e}")
+        sys.exit(1)
+
+    tok, mode = build_tokenizer(sudachi_config_path)
+
+    results: List[tf_schema.Project] = []
+
+    for p in projects:
+        project_id = p.get("projectId")
+
+        # 各フィールドのトークン化とTF
+        field_objs: Dict[str, tf_schema.TfOfField] = {}
+        doc_tf_counter: Counter = Counter()
+        doc_token_set: set = set()
+
+        for field in target_fields:
+            text = p.get(field) or ""
+            toks = tokenize(str(text), tok, mode, target_pos_l1, ban_list, stopwords)
+            tf = Counter(toks)
+            field_objs[field] = tf_schema.TfOfField(len=len(toks), tf=dict(tf))
+            doc_tf_counter.update(tf)
+            doc_token_set.update(tf.keys())
+
+        # スキーマ Fields へ詰める（未定義フィールドは長さ0/空dictで埋める）
+        def get_field(name: str) -> tf_schema.TfOfField:
+            return field_objs.get(name, tf_schema.TfOfField(len=0, tf={}))
+
+        fields_obj = tf_schema.Fields(
+            title=get_field("title"),
+            organization=get_field("organization"),
+            reading=get_field("reading"),
+            description=get_field("description"),
+            prComment=get_field("prComment"),
+            prCommentLong=get_field("prCommentLong"),
+        )
+
+        project_entry = tf_schema.Project(
+            projectId=project_id,
+            fields=fields_obj,
+            tf=dict(doc_tf_counter),
+            # tokens はユニーク語彙の存在フラグ(1)とする
+            tokens={t: 1 for t in sorted(doc_token_set)},
+        )
+        results.append(project_entry)
+
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    json_dumps([r.model_dump() for r in results], output_path)
+    log.info(f"tf_token.json を出力しました: {output_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/6_build_word_embeddings.py

@@ -0,0 +1,208 @@
+import os
+import sys
+import json
+import math
+from typing import Dict, List, Tuple
+
+import numpy as np
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from utils.logger import setup_logger
+from utils.json import get_file_path_from_config, field_getter, json_dumps
+
+log = setup_logger(__name__)
+
+
+def load_configs():
+    files = field_getter("config/files.json")
+    search = field_getter("config/search_model.json")
+
+    paths = {
+        "tf_token": files("bm25.tf_token"),
+        "bm25_meta": files("bm25.bm25_meta"),
+        "fasttext_vec": files("embeddings.fasttext_vec"),
+        "word_vocab": files("embeddings.word_vocab"),
+        "word_vectors": files("embeddings.word_vectors"),
+        "doc_vectors": files("embeddings.doc_vectors"),
+    }
+    field_weights: Dict[str, float] = search("bm25f.field_weights")
+    target_fields: List[str] = search("target_fields")
+
+    return paths, field_weights, target_fields
+
+
+def read_vocab_from_tf_token(tf_token_path: str) -> Tuple[List[dict], set[str]]:
+    with open(tf_token_path, encoding="utf-8") as f:
+        docs = json.load(f)
+    vocab: set[str] = set()
+    for d in docs:
+        # doc全体tfから語彙を得る
+        for t in (d.get("tf") or {}).keys():
+            vocab.add(t)
+    return docs, vocab
+
+
+def stream_fasttext_vec(vec_path: str, vocab: set[str]) -> Tuple[Dict[str, int], np.ndarray]:
+    """fastText .vec から、必要語彙のみ抽出してベクトル行列を返す。"""
+    token_to_idx: Dict[str, int] = {}
+    vectors: List[np.ndarray] = []
+
+    dim = None
+    kept = 0
+    with open(vec_path, encoding="utf-8", errors="ignore") as f:
+        header = f.readline()
+        # ヘッダ行は "<count> <dim>" のことが多い
+        try:
+            parts = header.strip().split()
+            if len(parts) >= 2 and parts[0].isdigit():
+                dim = int(parts[1])
+        except Exception:
+            pass
+
+        for line in f:
+            sp = line.rstrip().split(" ")
+            if len(sp) < 2:
+                continue
+            token = sp[0]
+            if token not in vocab:
+                continue
+            vec_vals = sp[1:]
+            if dim is None:
+                dim = len(vec_vals)
+            if len(vec_vals) != dim:
+                continue
+            try:
+                v = np.asarray([float(x) for x in vec_vals], dtype=np.float32)
+            except ValueError:
+                continue
+            # L2正規化
+            norm = np.linalg.norm(v)
+            if norm > 0:
+                v = v / norm
+            token_to_idx[token] = kept
+            vectors.append(v)
+            kept += 1
+
+    if dim is None:
+        raise RuntimeError(".vec の次元を特定できませんでした")
+
+    if not vectors:
+        log.warning("語彙に一致するベクトルが見つかりませんでした")
+        arr = np.zeros((0, dim), dtype=np.float32)
+        return token_to_idx, arr
+
+    arr = np.vstack(vectors).astype(np.float32)
+    return token_to_idx, arr
+
+
+def build_doc_vectors(
+    docs: List[dict],
+    token_to_idx: Dict[str, int],
+    word_vecs: np.ndarray,
+    idf: Dict[str, float],
+    field_weights: Dict[str, float],
+    target_fields: List[str],
+) -> np.ndarray:
+    dim = word_vecs.shape[1] if word_vecs.size > 0 else 300
+    doc_mat = np.zeros((len(docs), dim), dtype=np.float32)
+
+    for i, d in enumerate(docs):
+        accum = np.zeros((dim,), dtype=np.float32)
+        w_sum = 0.0
+
+        fields = (d.get("fields") or {})
+        for field in target_fields:
+            field_obj = fields.get(field) or {}
+            tf = field_obj.get("tf") or {}
+            f_weight = float(field_weights.get(field, 1.0))
+            if f_weight <= 0:
+                continue
+            for t, cnt in tf.items():
+                idx = token_to_idx.get(t)
+                if idx is None:
+                    continue
+                idf_t = float(idf.get(t, 0.0))
+                w = f_weight * idf_t * float(cnt)
+                if w <= 0:
+                    continue
+                accum += word_vecs[idx] * w
+                w_sum += w
+
+        if w_sum > 0:
+            vec = accum / w_sum
+            # L2正規化
+            n = np.linalg.norm(vec)
+            if n > 0:
+                vec = vec / n
+            doc_mat[i] = vec.astype(np.float32)
+        else:
+            # ベクトルなしの場合はゼロベクトル
+            doc_mat[i] = np.zeros((dim,), dtype=np.float32)
+
+    return doc_mat
+
+
+def main():
+    log.info("語彙/文書ベクトル(word_vocab.json, word_vectors.npz, doc_vectors.npy)を生成します")
+    try:
+        paths, field_weights, target_fields = load_configs()
+    except Exception as e:
+        log.error(f"設定の読み込みに失敗しました: {e}")
+        sys.exit(1)
+
+    tf_token_path = paths["tf_token"]
+    bm25_meta_path = paths["bm25_meta"]
+    fasttext_vec_path = paths["fasttext_vec"]
+
+    if not os.path.exists(fasttext_vec_path):
+        log.warning(f".vec が見つかりません: {fasttext_vec_path}")
+        log.warning("Step 6 をスキップします")
+        sys.exit(0)
+
+    try:
+        docs, vocab = read_vocab_from_tf_token(tf_token_path)
+    except Exception as e:
+        log.error(f"tf_token.jsonの読み込みに失敗しました: {e}")
+        sys.exit(1)
+
+    try:
+        with open(bm25_meta_path, encoding="utf-8") as f:
+            bm25_meta = json.load(f)
+            idf = bm25_meta.get("idf", {})
+    except Exception as e:
+        log.error(f"bm25_meta.jsonの読み込みに失敗しました: {e}")
+        sys.exit(1)
+
+    log.info(f"コーパス語彙数: {len(vocab)}")
+    token_to_idx, word_vecs = stream_fasttext_vec(fasttext_vec_path, vocab)
+    log.info(f"抽出済み語彙ベクトル数: {word_vecs.shape[0]}")
+
+    # 語彙インデックスの安定化（token_to_idxは追加順次第なのでソート）
+    sorted_tokens = sorted(token_to_idx.keys())
+    remap = {t: i for i, t in enumerate(sorted_tokens)}
+    remapped_vecs = np.zeros_like(word_vecs)
+    for t, old_i in token_to_idx.items():
+        new_i = remap[t]
+        remapped_vecs[new_i] = word_vecs[old_i]
+    token_to_idx = remap
+    word_vecs = remapped_vecs
+
+    # 文書ベクトル
+    doc_mat = build_doc_vectors(docs, token_to_idx, word_vecs, idf, field_weights, target_fields)
+
+    # 出力
+    os.makedirs(os.path.dirname(paths["word_vocab"]), exist_ok=True)
+    json_dumps(token_to_idx, paths["word_vocab"])  # 語→index
+    # 圧縮npz
+    np.savez_compressed(paths["word_vectors"], vectors=word_vecs)
+    np.save(paths["doc_vectors"], doc_mat)
+
+    log.info(f"word_vocab.json: {paths['word_vocab']}")
+    log.info(f"word_vectors.npz: {paths['word_vectors']}")
+    log.info(f"doc_vectors.npy: {paths['doc_vectors']}")
+
+
+if __name__ == "__main__":
+    main()
+

scripts/build_all.py

@@ -0,0 +1,53 @@
+import os
+import sys
+import subprocess
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from utils.logger import setup_logger
+from utils.json import field_getter
+
+log = setup_logger(__name__)
+
+
+def run_step(cmd: list[str], allow_fail: bool = False):
+    log.info("==> %s", " ".join(cmd))
+    try:
+        subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError as e:
+        if allow_fail:
+            log.warning(f"step失敗を無視します: {cmd} (code={e.returncode})")
+        else:
+            raise
+
+
+def main():
+    files = field_getter("config/files.json")
+
+    # Step 1: Sudachi user dict (optional but recommended before tokenization)
+    run_step([sys.executable, "scripts/1_build_dict.py"], allow_fail=True)
+
+    # Step 2: projects.json
+    run_step([sys.executable, "scripts/2_create_projects_data.py"])
+
+    # Step 3: synonyms cache
+    run_step([sys.executable, "scripts/3_build_synonyms_from_sudachi.py"])  # idempotent
+
+    # Step 4: bm25 meta
+    run_step([sys.executable, "scripts/4_prepare_bm25f_meta.py"])  # needs projects.json
+
+    # Step 5: tf_token
+    run_step([sys.executable, "scripts/5_prepare_tf_token.py"])  # needs projects.json
+
+    # Step 6: embeddings (.vec がある場合のみ)
+    vec_path = files("embeddings.fasttext_vec")
+    if os.path.exists(vec_path):
+        run_step([sys.executable, "scripts/6_build_word_embeddings.py"])  # needs tf_token, bm25_meta
+    else:
+        log.info(".vec が見つからないため Step 6 をスキップします: %s", vec_path)
+
+    log.info("全ステップ完了")
+
+
+if __name__ == "__main__":
+    main()