codex生成: bm35fとword_simのバランスを調整

api/main.py

@@ -100,12 +100,15 @@ def search(request: SearchRequest):
     if not request.query:
         raise HTTPException(status_code=400, detail="Query cannot be empty")
 
-    pairs = engine.search(request.query, debug=request.debug)
-    ids = [pid for pid, _ in pairs]
+    result = engine.search(request.query, debug=request.debug)
     if request.debug:
-        # Return scores as well for tuning. Bypass response_model filtering.
+        pairs, diag = result  # type: ignore
+        ids = [pid for pid, _ in pairs]
         return JSONResponse(content={
             "projectIds": ids,
             "scores": [{"projectId": pid, "score": float(score)} for pid, score in pairs],
+            "details": diag.get("details", []),
         })
+    pairs = result  # type: ignore
+    ids = [pid for pid, _ in pairs]
     return schema_projects.ProjectIds(projectIds=ids)

api/search/engine.py

@@ -25,6 +25,7 @@ class SearchConfig:
     word_sim_enable: bool
     word_sim_alpha: float
     word_sim_topk_k: int
+    word_sim_rerank: str
     query_subword_enable: bool
     query_subword_path: str
     query_subword_oov_weight: float
@@ -95,6 +96,7 @@ class SearchEngine:
             word_sim_enable=bool(search("word_sim.enable")),
             word_sim_alpha=float(search("word_sim.alpha")),
             word_sim_topk_k=int(search("word_sim.topk_k", 3)),
+            word_sim_rerank=(search("word_sim.rerank", "pair_avg") or "pair_avg").lower(),
             query_subword_enable=bool(search("query_subword.enable")),
             query_subword_path=search("query_subword.path"),
             query_subword_oov_weight=float(search("query_subword.oov_weight")),
@@ -275,7 +277,7 @@ class SearchEngine:
                 return None, True
         return None, True
 
-    def _word_sim_scores(self, terms: List[str]) -> Optional[np.ndarray]:
+    def _word_sim_scores_topk(self, terms: List[str]) -> Optional[np.ndarray]:
         if not self.cfg.word_sim_enable:
             return None
         if self.word_vectors is None:
@@ -347,28 +349,68 @@ class SearchEngine:
             sims_all[i] = float(top_vals.mean()) if top_vals.size else 0.0
         return sims_all
 
+    def _word_sim_scores_pairavg(self, terms: List[str]) -> Optional[np.ndarray]:
+        if not self.cfg.word_sim_enable:
+            return None
+        if self.word_vectors is None:
+            return None
+        # Build query term vectors (no weighting for pair-avg, simple mean over all pairs)
+        vecs = []
+        for t in terms:
+            v, _ = self._get_token_vector(t)
+            if v is None:
+                continue
+            vecs.append(v)
+        if not vecs:
+            return None
+        Vq = np.stack(vecs).astype(np.float32)  # Tq x D
+
+        n_docs = len(self.tf_token_docs)
+        sims_all = np.zeros((n_docs,), dtype=np.float32)
+        for i, d in enumerate(self.tf_token_docs):
+            fields = d.get("fields") or {}
+            doc_terms = set()
+            for fname in self.cfg.target_fields:
+                fobj = (fields.get(fname) or {})
+                tf = (fobj.get("tf") or {})
+                doc_terms.update(tf.keys())
+            if not doc_terms:
+                sims_all[i] = 0.0
+                continue
+            Vd_list = []
+            for t in doc_terms:
+                idx = self.word_vocab.get(t)
+                if idx is None:
+                    continue
+                Vd_list.append(self.word_vectors[idx])
+            if not Vd_list:
+                sims_all[i] = 0.0
+                continue
+            Vd = np.stack(Vd_list).astype(np.float32)  # Td x D
+            M = Vd @ Vq.T  # Td x Tq
+            M = np.maximum(M, 0.0)
+            sims_all[i] = float(M.mean()) if M.size else 0.0
+        return sims_all
+
     # ----- Public API -----
     def search(
         self,
         query: str,
         debug: bool = False,
-    ) -> List[Tuple[str, float]]:
+    ) -> List[Tuple[str, float]] | Tuple[List[Tuple[str, float]], Dict[str, Any]]:
         terms = self._tokenize(query)
         if self.cfg.synonyms_enable:
             terms = self._expand_synonyms(terms)
 
         # BM25F
         bm25 = self._bm25f_scores(terms)
-        score = bm25.copy()
 
-        # word sim
-        ws = self._word_sim_scores(terms)
-        if ws is not None:
-            a = float(self.cfg.word_sim_alpha)
-            score = a * score + (1.0 - a) * ws
-        else:
-            # use zero vector for filtering logic
-            ws = np.zeros_like(score)
+        # word sim (filtering): top-k pooling
+        ws_filter = self._word_sim_scores_topk(terms)
+        if ws_filter is None:
+            ws_filter = np.zeros_like(bm25)
+        a = float(self.cfg.word_sim_alpha)
+        fused_filter = a * bm25 + (1.0 - a) * ws_filter
 
         # Organization/reading auto-boost based on raw query substring match
         qn = normalize_text_for_org(query)
@@ -392,17 +434,18 @@ class SearchEngine:
                 + prefix.astype(np.float32) * float(self.cfg.org_boost_prefix)
                 + substr.astype(np.float32) * float(self.cfg.org_boost_substring)
             )
-            score = score + boost
+            pass
 
         # collect results
         ids = [d.get("projectId") for d in self.projects]
         # Filtering to reduce false positives while keeping recall
         # Relative threshold anchored to the top fused score
-        top = float(np.max(score)) if score.size > 0 else 0.0
+        score_with_boost = fused_filter + boost
+        top = float(np.max(score_with_boost)) if score_with_boost.size > 0 else 0.0
         rel_cut = top * float(self.cfg.fused_rel_top_ratio) if top > 0 else self.cfg.fused_min
         fused_cut = max(float(self.cfg.fused_min), rel_cut)
-        keep = ((bm25 >= self.cfg.bm25_min) | (ws >= self.cfg.word_sim_min) | (score >= self.cfg.fused_min)) & (score >= fused_cut)
-        order = np.argsort(-score)  # descending by fused
+        keep = ((bm25 >= self.cfg.bm25_min) | (ws_filter >= self.cfg.word_sim_min) | (score_with_boost >= self.cfg.fused_min)) & (score_with_boost >= fused_cut)
+        order = np.argsort(-score_with_boost)  # descending by fused
         selected_idx: List[int] = []
         for i in order:
             if keep[i]:
@@ -411,17 +454,39 @@ class SearchEngine:
                 break
         # Single-step fallback: if zero, relax the relative cut and use absolute thresholds only
         if len(selected_idx) == 0:
-            keep2 = (bm25 >= self.cfg.bm25_min) | (ws >= self.cfg.word_sim_min) | (score >= self.cfg.fused_min)
+            keep2 = (bm25 >= self.cfg.bm25_min) | (ws_filter >= self.cfg.word_sim_min) | (score_with_boost >= self.cfg.fused_min)
             for i in order:
                 if keep2[i]:
                     selected_idx.append(int(i))
                 if len(selected_idx) >= self.cfg.max_results:
                     break
-        pairs = [(ids[i], float(score[i])) for i in selected_idx]
+        # Rerank with pair-avg word similarity (if enabled)
+        ws_rerank = None
+        if self.cfg.word_sim_rerank == "pair_avg":
+            ws_rerank = self._word_sim_scores_pairavg(terms)
+        if ws_rerank is None:
+            ws_rerank = ws_filter
+        fused_rerank = a * bm25 + (1.0 - a) * ws_rerank
+        final_scores = fused_rerank + boost
+        pairs = [(ids[i], float(final_scores[i])) for i in selected_idx]
 
         # sort
         pairs.sort(key=lambda x: (-x[1], x[0]))
-        return pairs
+        if not debug:
+            return pairs
+        # build debug details for selected docs
+        details = []
+        for i in selected_idx:
+            details.append({
+                "projectId": ids[i],
+                "bm25": float(bm25[i]),
+                "ws_filter_topk": float(ws_filter[i]),
+                "ws_rerank_pairavg": float(ws_rerank[i]) if ws_rerank is not None else None,
+                "org_boost": float(boost[i]),
+                "fused_filter": float(fused_filter[i]),
+                "fused_final": float(final_scores[i]),
+            })
+        return pairs, {"details": details}
 
     def get_projects(self) -> List[Dict[str, Any]]:
         return self.projects

config/search_model.json

@@ -45,8 +45,9 @@
   "word_sim": {
     "enable": true,
     "mode": "topk",
-    "alpha": 0.7,
-    "topk_k": 3
+    "alpha": 0.3,
+    "topk_k": 3,
+    "rerank": "pair_avg"
   },
   "query_subword": {
     "enable": true,