Merge pull request #10 from chibafes-dev/codex/substring-engine

{api → app}/main.py

@@ -12,9 +12,10 @@ from pydantic import BaseModel
 from dotenv import load_dotenv
 
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+sys.path.append(os.path.dirname(__file__))
 
 from utils.logger import setup_logger
-from api.search.engine import SearchEngine
+from search.engine import SearchEngine
 import schemas.projects as schema_projects
 
 log = setup_logger(__name__)

{api → app}/search/engine.py

@@ -1,8 +1,9 @@
 import json
 import os
 import sys
+import unicodedata
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Set, Tuple
 
 import numpy as np
 from sudachipy import dictionary, tokenizer
@@ -68,8 +69,10 @@ class SearchEngine:
         # Data
         self.projects: List[Dict[str, Any]] = []
         self.project_map: Dict[str, Dict[str, Any]] = {}
+        self.project_idx: Dict[str, int] = {}
         self.org_norms: Dict[str, str] = {}
         self.reading_norms: Dict[str, str] = {}
+        self.substring_index: Dict[str, List[str]] = {}
 
         # BM25F assets
         self.idf: Dict[str, float] = {}
@@ -146,10 +149,22 @@ class SearchEngine:
         except Exception:
             pass
 
+        # Substring index for organization substring lookup
+        substring_index_path = files("substring.substring_index")
+        if os.path.exists(substring_index_path):
+            try:
+                with open(substring_index_path, encoding="utf-8") as f:
+                    self.substring_index = json.load(f)
+            except Exception as e:
+                log.warning(f"failed to load substring_index: {e}")
+        else:
+            self.substring_index = {}
+
         # Projects
         with open(files("projects.projects_json"), encoding="utf-8") as f:
             self.projects = json.load(f)
         self.project_map = {p["projectId"]: p for p in self.projects}
+        self.project_idx = {p["projectId"]: idx for idx, p in enumerate(self.projects)}
         self.org_norms = {
             p["projectId"]: normalize_text_for_org(p.get("organization") or "")
             for p in self.projects
@@ -238,6 +253,80 @@ class SearchEngine:
         max_q = int(self.cfg.syn_limits.get("max_query_variants", 5))
         return expanded[: max_q * max_exp + len(terms)]
 
+    @staticmethod
+    def _katakana_to_hiragana(text: str) -> str:
+        if not text:
+            return ""
+        chars: List[str] = []
+        for ch in text:
+            code = ord(ch)
+            if 0x30A1 <= code <= 0x30F6:
+                chars.append(chr(code - 0x60))
+            else:
+                chars.append(ch)
+        return "".join(chars)
+
+    def _normalize_substring_token(self, token: str) -> str:
+        if not token:
+            return ""
+        try:
+            token_nfkc = unicodedata.normalize("NFKC", token)
+        except Exception:
+            token_nfkc = token
+
+        readings: List[str] = []
+        if self.tokenizer is not None:
+            try:
+                for m in self.tokenizer.tokenize(token_nfkc, self.mode):
+                    reading = m.reading_form()
+                    if not reading or reading == "*":
+                        reading = m.normalized_form()
+                    if reading:
+                        readings.append(reading)
+            except Exception:
+                readings = []
+
+        reading = "".join(readings) if readings else token_nfkc
+        lowered = reading.lower()
+        hira = self._katakana_to_hiragana(lowered)
+
+        normalized_chars: List[str] = []
+        for ch in hira:
+            if ch in ("\u0020", "\u3000"):
+                continue
+            category = unicodedata.category(ch)
+            if category.startswith("P") or category.startswith("S"):
+                if ch != "ー":
+                    continue
+            normalized_chars.append(ch)
+        return "".join(normalized_chars)
+
+    def _normalize_substring_terms(self, query: str) -> List[str]:
+        if not query:
+            return []
+        try:
+            normalized_query = unicodedata.normalize("NFKC", query)
+        except Exception:
+            normalized_query = query
+
+        out: List[str] = []
+        for raw in normalized_query.split():
+            term = self._normalize_substring_token(raw)
+            if term:
+                out.append(term)
+        return out
+
+    def _substring_match_project_ids(self, query: str) -> Set[str]:
+        if not self.substring_index:
+            return set()
+        terms = self._normalize_substring_terms(query)
+        matches: Set[str] = set()
+        for term in terms:
+            if len(term) < 2:
+                continue
+            matches.update(self.substring_index.get(term, []))
+        return matches
+
     # ----- BM25F -----
     def _bm25f_scores(self, terms: List[str]) -> np.ndarray:
         N = len(self.tf_token_docs)
@@ -419,6 +508,17 @@ class SearchEngine:
         if self.cfg.synonyms_enable:
             terms = self._expand_synonyms(terms)
 
+        substring_hits = self._substring_match_project_ids(query)
+        substring_idx_set: Set[int] = set()
+        substring_mask = np.zeros((len(self.projects),), dtype=bool)
+        if substring_hits:
+            for pid in substring_hits:
+                idx = self.project_idx.get(pid)
+                if idx is None:
+                    continue
+                substring_idx_set.add(idx)
+                substring_mask[idx] = True
+
         # BM25F
         bm25 = self._bm25f_scores(terms)
 
@@ -474,24 +574,42 @@ class SearchEngine:
             | (ws_filter >= self.cfg.word_sim_min)
             | (score_with_boost >= self.cfg.fused_min)
         ) & (score_with_boost >= fused_cut)
+        if substring_idx_set:
+            keep = keep | substring_mask
         order = np.argsort(-score_with_boost)  # descending by fused
         selected_idx: List[int] = []
+        selected_idx_set: Set[int] = set()
+        substring_sorted = sorted(substring_idx_set, key=lambda i: -score_with_boost[i])
+        for idx in substring_sorted:
+            selected_idx.append(int(idx))
+            selected_idx_set.add(int(idx))
+        non_sub_count = 0
         for i in order:
-            if keep[i]:
-                selected_idx.append(int(i))
-            if len(selected_idx) >= self.cfg.max_results:
+            idx = int(i)
+            if idx in selected_idx_set:
+                continue
+            if keep[idx]:
+                selected_idx.append(idx)
+                selected_idx_set.add(idx)
+                non_sub_count += 1
+            if non_sub_count >= self.cfg.max_results:
                 break
         # Single-step fallback: if zero, relax the relative cut and use absolute thresholds only
-        if len(selected_idx) == 0:
+        if not selected_idx_set:
             keep2 = (
                 (bm25 >= self.cfg.bm25_min)
                 | (ws_filter >= self.cfg.word_sim_min)
                 | (score_with_boost >= self.cfg.fused_min)
             )
             for i in order:
-                if keep2[i]:
-                    selected_idx.append(int(i))
-                if len(selected_idx) >= self.cfg.max_results:
+                idx = int(i)
+                if idx in selected_idx_set:
+                    continue
+                if keep2[idx]:
+                    selected_idx.append(idx)
+                    selected_idx_set.add(idx)
+                    non_sub_count += 1
+                if non_sub_count >= self.cfg.max_results:
                     break
         # Rerank with pair-avg word similarity (if enabled)
         ws_rerank = None
@@ -522,6 +640,7 @@ class SearchEngine:
                     if ws_rerank is not None
                     else None,
                     "org_boost": float(boost[i]) if boost_enabled else None,
+                    "matched_substring": bool(substring_mask[i]),
                     "fused_filter": float(fused_filter[i]),
                     "fused_final": float(final_scores[i]),
                 }