codex生成: topkのみに

api/search/engine.py

@@ -23,7 +23,6 @@ class SearchConfig:
     syn_limits: Dict[str, int]
     banlist: List[str]
     word_sim_enable: bool
-    word_sim_mode: str
     word_sim_alpha: float
     word_sim_topk_k: int
     query_subword_enable: bool
@@ -76,7 +75,6 @@ class SearchEngine:
         # Vectors
         self.word_vocab: Dict[str, int] = {}
         self.word_vectors: Optional[np.ndarray] = None
-        self.doc_vectors: Optional[np.ndarray] = None
         self.ft_model = None
 
     # ----- Init / Load -----
@@ -95,7 +93,6 @@ class SearchEngine:
             syn_limits=search("synonyms.limits"),
             banlist=search("synonyms.banlist"),
             word_sim_enable=bool(search("word_sim.enable")),
-            word_sim_mode=(search("word_sim.mode") or "soft").lower(),
             word_sim_alpha=float(search("word_sim.alpha")),
             word_sim_topk_k=int(search("word_sim.topk_k", 3)),
             query_subword_enable=bool(search("query_subword.enable")),
@@ -167,12 +164,7 @@ class SearchEngine:
         except Exception as e:
             log.warning(f"word vectors not ready: {e}")
 
-        try:
-            doc_vec_path = files("embeddings.doc_vectors")
-            if os.path.exists(doc_vec_path):
-                self.doc_vectors = np.load(doc_vec_path)
-        except Exception as e:
-            log.warning(f"doc vectors not ready: {e}")
+        # doc_vectors.npy は topk 方式では不要
 
         # fastText OOV
         if self.cfg.query_subword_enable and self.cfg.query_subword_path and os.path.exists(self.cfg.query_subword_path):
@@ -286,7 +278,7 @@ class SearchEngine:
     def _word_sim_scores(self, terms: List[str]) -> Optional[np.ndarray]:
         if not self.cfg.word_sim_enable:
             return None
-        if self.word_vectors is None or self.doc_vectors is None or self.doc_vectors.size == 0:
+        if self.word_vectors is None:
             return None
 
         # Build per-term vectors with weights (IDF; OOV down-weighted)
@@ -310,71 +302,50 @@ class SearchEngine:
         V = np.stack(vecs).astype(np.float32)  # T x D
         W = np.asarray(weights, dtype=np.float32)  # T
 
-        # Mode: average query vector then cosine
-        if self.cfg.word_sim_mode in ("avg", "soft"):
-            q = np.average(V, axis=0, weights=W)
-            n = np.linalg.norm(q)
-            if n > 0:
-                q = q / n
-            if not np.all(np.isfinite(q)):
-                return None
-            sims = self.doc_vectors @ q.astype(np.float32)
-            return sims
-
-        # Mode: top-k pooling over term-term cosine contributions (query terms x document terms)
-        if self.cfg.word_sim_mode == "topk":
-            k = max(1, int(self.cfg.word_sim_topk_k))
-            n_docs = len(self.tf_token_docs)
-            sims_all = np.zeros((n_docs,), dtype=np.float32)
-            # Pre-shape
-            Vq = V  # Tq x D (already float32, L2-normalized token vectors)
-            Wq = W  # Tq
-            for i, d in enumerate(self.tf_token_docs):
-                # collect unique doc tokens across target fields
-                fields = d.get("fields") or {}
-                doc_terms = set()
-                for fname in self.cfg.target_fields:
-                    fobj = (fields.get(fname) or {})
-                    tf = (fobj.get("tf") or {})
-                    doc_terms.update(tf.keys())
-                if not doc_terms:
-                    sims_all[i] = 0.0
-                    continue
-                # build matrix of doc term vectors
-                Vd_list = []
-                for t in doc_terms:
-                    idx = self.word_vocab.get(t)
-                    if idx is None:
-                        continue
-                    Vd_list.append(self.word_vectors[idx])
-                if not Vd_list:
-                    sims_all[i] = 0.0
-                    continue
-                Vd = np.stack(Vd_list).astype(np.float32)  # Td x D
-                # pairwise cosine (since vectors are normalized)
-                M = Vd @ Vq.T  # Td x Tq
-                # weight by query term importance and clip negatives
-                if Wq.size:
-                    M = M * Wq[None, :]
-                M = np.maximum(M, 0.0)
-                # take global top-k across all pairs
-                Td, Tq = M.shape
-                total = Td * Tq
-                kk = min(k, total) if total > 0 else 0
-                if kk == 0:
-                    sims_all[i] = 0.0
+        # top-k pooling over term-term cosine contributions (query terms x document terms)
+        k = max(1, int(self.cfg.word_sim_topk_k))
+        n_docs = len(self.tf_token_docs)
+        sims_all = np.zeros((n_docs,), dtype=np.float32)
+        Vq = V  # Tq x D (normalized)
+        Wq = W  # Tq
+        for i, d in enumerate(self.tf_token_docs):
+            fields = d.get("fields") or {}
+            doc_terms = set()
+            for fname in self.cfg.target_fields:
+                fobj = (fields.get(fname) or {})
+                tf = (fobj.get("tf") or {})
+                doc_terms.update(tf.keys())
+            if not doc_terms:
+                sims_all[i] = 0.0
+                continue
+            Vd_list = []
+            for t in doc_terms:
+                idx = self.word_vocab.get(t)
+                if idx is None:
                     continue
-                flat = M.reshape(-1)
-                if kk == total:
-                    top_vals = flat
-                else:
-                    idxk = np.argpartition(flat, -kk)[-kk:]
-                    top_vals = flat[idxk]
-                sims_all[i] = float(top_vals.mean()) if top_vals.size else 0.0
-            return sims_all
-
-        # Fallback: disable if unknown mode
-        return None
+                Vd_list.append(self.word_vectors[idx])
+            if not Vd_list:
+                sims_all[i] = 0.0
+                continue
+            Vd = np.stack(Vd_list).astype(np.float32)  # Td x D
+            M = Vd @ Vq.T  # Td x Tq
+            if Wq.size:
+                M = M * Wq[None, :]
+            M = np.maximum(M, 0.0)
+            Td, Tq = M.shape
+            total = Td * Tq
+            kk = min(k, total) if total > 0 else 0
+            if kk == 0:
+                sims_all[i] = 0.0
+                continue
+            flat = M.reshape(-1)
+            if kk == total:
+                top_vals = flat
+            else:
+                idxk = np.argpartition(flat, -kk)[-kk:]
+                top_vals = flat[idxk]
+            sims_all[i] = float(top_vals.mean()) if top_vals.size else 0.0
+        return sims_all
 
     # ----- Public API -----
     def search(

docs/files.md

@@ -13,5 +13,3 @@
         - `scripts/6_build_word_embeddings.py`で生成
     - `word_vectors.npz`
         - `scripts/6_build_word_embeddings.py`で生成
-    - `doc_vectors.npy`
-        - `scripts/6_build_word_embeddings.py`で生成

scripts/6_build_word_embeddings.py

@@ -20,16 +20,11 @@ def load_configs():
 
     paths = {
         "tf_token": files("bm25.tf_token"),
-        "bm25_meta": files("bm25.bm25_meta"),
         "fasttext_vec": files("embeddings.fasttext_vec"),
         "word_vocab": files("embeddings.word_vocab"),
         "word_vectors": files("embeddings.word_vectors"),
-        "doc_vectors": files("embeddings.doc_vectors"),
     }
-    field_weights: Dict[str, float] = search("bm25f.field_weights")
-    target_fields: List[str] = search("target_fields")
-
-    return paths, field_weights, target_fields
+    return paths
 
 
 def read_vocab_from_tf_token(tf_token_path: str) -> Tuple[List[dict], set[str]]:
@@ -96,63 +91,18 @@ def stream_fasttext_vec(vec_path: str, vocab: set[str]) -> Tuple[Dict[str, int],
     return token_to_idx, arr
 
 
-def build_doc_vectors(
-    docs: List[dict],
-    token_to_idx: Dict[str, int],
-    word_vecs: np.ndarray,
-    idf: Dict[str, float],
-    field_weights: Dict[str, float],
-    target_fields: List[str],
-) -> np.ndarray:
-    dim = word_vecs.shape[1] if word_vecs.size > 0 else 300
-    doc_mat = np.zeros((len(docs), dim), dtype=np.float32)
-
-    for i, d in enumerate(docs):
-        accum = np.zeros((dim,), dtype=np.float32)
-        w_sum = 0.0
-
-        fields = (d.get("fields") or {})
-        for field in target_fields:
-            field_obj = fields.get(field) or {}
-            tf = field_obj.get("tf") or {}
-            f_weight = float(field_weights.get(field, 1.0))
-            if f_weight <= 0:
-                continue
-            for t, cnt in tf.items():
-                idx = token_to_idx.get(t)
-                if idx is None:
-                    continue
-                idf_t = float(idf.get(t, 0.0))
-                w = f_weight * idf_t * float(cnt)
-                if w <= 0:
-                    continue
-                accum += word_vecs[idx] * w
-                w_sum += w
-
-        if w_sum > 0:
-            vec = accum / w_sum
-            # L2正規化
-            n = np.linalg.norm(vec)
-            if n > 0:
-                vec = vec / n
-            doc_mat[i] = vec.astype(np.float32)
-        else:
-            # ベクトルなしの場合はゼロベクトル
-            doc_mat[i] = np.zeros((dim,), dtype=np.float32)
-
-    return doc_mat
+# top-k モードでは文書ベクトルは不要
 
 
 def main():
     log.info("語彙/文書ベクトル(word_vocab.json, word_vectors.npz, doc_vectors.npy)を生成します")
     try:
-        paths, field_weights, target_fields = load_configs()
+        paths = load_configs()
     except Exception as e:
         log.error(f"設定の読み込みに失敗しました: {e}")
         sys.exit(1)
 
     tf_token_path = paths["tf_token"]
-    bm25_meta_path = paths["bm25_meta"]
     fasttext_vec_path = paths["fasttext_vec"]
 
     if not os.path.exists(fasttext_vec_path):
@@ -166,14 +116,6 @@ def main():
         log.error(f"tf_token.jsonの読み込みに失敗しました: {e}")
         sys.exit(1)
 
-    try:
-        with open(bm25_meta_path, encoding="utf-8") as f:
-            bm25_meta = json.load(f)
-            idf = bm25_meta.get("idf", {})
-    except Exception as e:
-        log.error(f"bm25_meta.jsonの読み込みに失敗しました: {e}")
-        sys.exit(1)
-
     log.info(f"コーパス語彙数: {len(vocab)}")
     token_to_idx, word_vecs = stream_fasttext_vec(fasttext_vec_path, vocab)
     log.info(f"抽出済み語彙ベクトル数: {word_vecs.shape[0]}")
@@ -188,21 +130,15 @@ def main():
     token_to_idx = remap
     word_vecs = remapped_vecs
 
-    # 文書ベクトル
-    doc_mat = build_doc_vectors(docs, token_to_idx, word_vecs, idf, field_weights, target_fields)
-
     # 出力
     os.makedirs(os.path.dirname(paths["word_vocab"]), exist_ok=True)
     json_dumps(token_to_idx, paths["word_vocab"])  # 語→index
     # 圧縮npz
     np.savez_compressed(paths["word_vectors"], vectors=word_vecs)
-    np.save(paths["doc_vectors"], doc_mat)
 
     log.info(f"word_vocab.json: {paths['word_vocab']}")
     log.info(f"word_vectors.npz: {paths['word_vectors']}")
-    log.info(f"doc_vectors.npy: {paths['doc_vectors']}")
 
 
 if __name__ == "__main__":
     main()
-