Merge pull request #11 from chibafes-dev/improve/issues-7

{api → app}/main.py

@@ -12,9 +12,10 @@ from pydantic import BaseModel
 from dotenv import load_dotenv
 
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+sys.path.append(os.path.dirname(__file__))
 
 from utils.logger import setup_logger
-from api.search.engine import SearchEngine
+from search.engine import SearchEngine
 import schemas.projects as schema_projects
 
 log = setup_logger(__name__)

{api → app}/search/engine.py

@@ -1,8 +1,9 @@
 import json
 import os
 import sys
+import unicodedata
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Set, Tuple
 
 import numpy as np
 from sudachipy import dictionary, tokenizer
@@ -68,8 +69,10 @@ class SearchEngine:
         # Data
         self.projects: List[Dict[str, Any]] = []
         self.project_map: Dict[str, Dict[str, Any]] = {}
+        self.project_idx: Dict[str, int] = {}
         self.org_norms: Dict[str, str] = {}
         self.reading_norms: Dict[str, str] = {}
+        self.substring_index: Dict[str, List[str]] = {}
 
         # BM25F assets
         self.idf: Dict[str, float] = {}
@@ -146,10 +149,22 @@ class SearchEngine:
         except Exception:
             pass
 
+        # Substring index for organization substring lookup
+        substring_index_path = files("substring.substring_index")
+        if os.path.exists(substring_index_path):
+            try:
+                with open(substring_index_path, encoding="utf-8") as f:
+                    self.substring_index = json.load(f)
+            except Exception as e:
+                log.warning(f"failed to load substring_index: {e}")
+        else:
+            self.substring_index = {}
+
         # Projects
         with open(files("projects.projects_json"), encoding="utf-8") as f:
             self.projects = json.load(f)
         self.project_map = {p["projectId"]: p for p in self.projects}
+        self.project_idx = {p["projectId"]: idx for idx, p in enumerate(self.projects)}
         self.org_norms = {
             p["projectId"]: normalize_text_for_org(p.get("organization") or "")
             for p in self.projects
@@ -238,6 +253,82 @@ class SearchEngine:
         max_q = int(self.cfg.syn_limits.get("max_query_variants", 5))
         return expanded[: max_q * max_exp + len(terms)]
 
+    @staticmethod
+    def _katakana_to_hiragana(text: str) -> str:
+        if not text:
+            return ""
+        chars: List[str] = []
+        for ch in text:
+            code = ord(ch)
+            if 0x30A1 <= code <= 0x30F6:
+                chars.append(chr(code - 0x60))
+            else:
+                chars.append(ch)
+        return "".join(chars)
+
+    def _normalize_substring_token(self, token: str) -> str:
+        if not token:
+            return ""
+        try:
+            token_nfkc = unicodedata.normalize("NFKC", token)
+        except Exception:
+            token_nfkc = token
+
+        readings: List[str] = []
+        if self.tokenizer is not None:
+            try:
+                for m in self.tokenizer.tokenize(
+                    token_nfkc, tokenizer.Tokenizer.SplitMode.C
+                ):
+                    reading = m.reading_form()
+                    if not reading or reading == "*":
+                        reading = m.normalized_form()
+                    if reading:
+                        readings.append(reading)
+            except Exception:
+                readings = []
+
+        reading = "".join(readings) if readings else token_nfkc
+        lowered = reading.lower()
+        hira = self._katakana_to_hiragana(lowered)
+
+        normalized_chars: List[str] = []
+        for ch in hira:
+            if ch in ("\u0020", "\u3000"):
+                continue
+            category = unicodedata.category(ch)
+            if category.startswith("P") or category.startswith("S"):
+                if ch != "ー":
+                    continue
+            normalized_chars.append(ch)
+        return "".join(normalized_chars)
+
+    def _normalize_substring_terms(self, query: str) -> List[str]:
+        if not query:
+            return []
+        try:
+            normalized_query = unicodedata.normalize("NFKC", query)
+        except Exception:
+            normalized_query = query
+
+        out: List[str] = []
+        for raw in normalized_query.split():
+            term = self._normalize_substring_token(raw)
+            if term:
+                out.append(term)
+        return out
+
+    def _substring_match_project_ids(self, query: str) -> Set[str]:
+        if not self.substring_index:
+            return set()
+        terms = self._normalize_substring_terms(query)
+        matches: Set[str] = set()
+        for term in terms:
+            if len(term) < 2:
+                continue
+            matches.update(self.substring_index.get(term, []))
+        return matches
+
     # ----- BM25F -----
     def _bm25f_scores(self, terms: List[str]) -> np.ndarray:
         N = len(self.tf_token_docs)
@@ -419,6 +510,17 @@ class SearchEngine:
         if self.cfg.synonyms_enable:
             terms = self._expand_synonyms(terms)
 
+        substring_hits = self._substring_match_project_ids(query)
+        substring_idx_set: Set[int] = set()
+        substring_mask = np.zeros((len(self.projects),), dtype=bool)
+        if substring_hits:
+            for pid in substring_hits:
+                idx = self.project_idx.get(pid)
+                if idx is None:
+                    continue
+                substring_idx_set.add(idx)
+                substring_mask[idx] = True
+
         # BM25F
         bm25 = self._bm25f_scores(terms)
 
@@ -432,6 +534,7 @@ class SearchEngine:
         # Organization/reading auto-boost based on raw query substring match
         qn = normalize_text_for_org(query)
         boost_enabled = len(qn) >= int(self.cfg.org_boost_min_len)
+        boost = np.zeros((len(self.projects),), dtype=np.float32)
         if boost_enabled:
             exact = np.zeros((len(self.projects),), dtype=bool)
             prefix = np.zeros_like(exact)
@@ -452,7 +555,6 @@ class SearchEngine:
                 + prefix.astype(np.float32) * float(self.cfg.org_boost_prefix)
                 + substr.astype(np.float32) * float(self.cfg.org_boost_substring)
             )
-            pass
 
         # collect results
         ids = [d.get("projectId") for d in self.projects]
@@ -474,24 +576,42 @@ class SearchEngine:
             | (ws_filter >= self.cfg.word_sim_min)
             | (score_with_boost >= self.cfg.fused_min)
         ) & (score_with_boost >= fused_cut)
+        if substring_idx_set:
+            keep = keep | substring_mask
         order = np.argsort(-score_with_boost)  # descending by fused
         selected_idx: List[int] = []
+        selected_idx_set: Set[int] = set()
+        substring_sorted = sorted(substring_idx_set, key=lambda i: -score_with_boost[i])
+        for idx in substring_sorted:
+            selected_idx.append(int(idx))
+            selected_idx_set.add(int(idx))
+        non_sub_count = 0
         for i in order:
-            if keep[i]:
-                selected_idx.append(int(i))
-            if len(selected_idx) >= self.cfg.max_results:
+            idx = int(i)
+            if idx in selected_idx_set:
+                continue
+            if keep[idx]:
+                selected_idx.append(idx)
+                selected_idx_set.add(idx)
+                non_sub_count += 1
+            if non_sub_count >= self.cfg.max_results:
                 break
         # Single-step fallback: if zero, relax the relative cut and use absolute thresholds only
-        if len(selected_idx) == 0:
+        if not selected_idx_set:
             keep2 = (
                 (bm25 >= self.cfg.bm25_min)
                 | (ws_filter >= self.cfg.word_sim_min)
                 | (score_with_boost >= self.cfg.fused_min)
             )
             for i in order:
-                if keep2[i]:
-                    selected_idx.append(int(i))
-                if len(selected_idx) >= self.cfg.max_results:
+                idx = int(i)
+                if idx in selected_idx_set:
+                    continue
+                if keep2[idx]:
+                    selected_idx.append(idx)
+                    selected_idx_set.add(idx)
+                    non_sub_count += 1
+                if non_sub_count >= self.cfg.max_results:
                     break
         # Rerank with pair-avg word similarity (if enabled)
         ws_rerank = None
@@ -510,20 +630,28 @@ class SearchEngine:
         pairs.sort(key=lambda x: (-x[1], x[0]))
         if not debug:
             return pairs
-        # build debug details for selected docs
+        # build debug details for all docs sorted by score
+        ranked_indices = sorted(
+            range(len(self.projects)),
+            key=lambda idx: (-float(final_scores[idx]), ids[idx]),
+        )
         details = []
-        for i in selected_idx:
+        for idx in ranked_indices:
+            project = self.projects[idx]
             details.append(
                 {
-                    "projectId": ids[i],
-                    "bm25": float(bm25[i]),
-                    "ws_filter_topk": float(ws_filter[i]),
-                    "ws_rerank_pairavg": float(ws_rerank[i])
+                    "projectId": ids[idx],
+                    "organization": project.get("organization"),
+                    "title": project.get("title"),
+                    "bm25": float(bm25[idx]),
+                    "ws_filter_topk": float(ws_filter[idx]),
+                    "ws_rerank_pairavg": float(ws_rerank[idx])
                     if ws_rerank is not None
                     else None,
-                    "org_boost": float(boost[i]) if boost_enabled else None,
-                    "fused_filter": float(fused_filter[i]),
-                    "fused_final": float(final_scores[i]),
+                    "org_boost": float(boost[idx]) if boost_enabled else None,
+                    "matched_substring": bool(substring_mask[idx]),
+                    "fused_filter": float(fused_filter[idx]),
+                    "fused_final": float(final_scores[idx]),
                 }
             )
         return pairs, {"details": details}

config/files.json

@@ -15,6 +15,10 @@
         "bm25_meta": "data/generated/bm25_meta.json",
         "tf_token": "data/generated/tf_token.json"
     },
+    "substring": {
+        "circle_names": "data/generated/circle_names.json",
+        "substring_index": "data/generated/substring_index.json"
+    },
     "embeddings": {
         "fasttext_vec": "resources/embeddings/cc.ja.300.vec",
         "fasttext_bin": "resources/embeddings/cc.ja.300.bin",

config/search_model.json

@@ -45,7 +45,7 @@
   "word_sim": {
     "enable": true,
     "mode": "topk",
-    "alpha": 0.3,
+    "alpha": 0.5,
     "topk_k": 3,
     "rerank": "pair_avg"
   },

scripts/3_build_synonyms_from_sudachi.py

@@ -12,7 +12,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 from utils.logger import setup_logger
 from utils.json import get_file_path_from_config, field_getter, json_dumps
 from utils.io import LineIteratorIO, comment_filtered_lines
-import schemas.projects as projects_schema
+from schemas.projects import Project
 
 # --- ロギングの設定 ---
 log = setup_logger(__name__)
@@ -65,7 +65,7 @@ def tokenize(text: str) -> list[str]:
     return tokens
 
 
-def get_corpus_vocab(projects: list[projects_schema.Project]) -> set[str]:
+def get_corpus_vocab(projects: list[Project]) -> set[str]:
     """プロジェクト全体から語彙セットを構築する"""
     vocab = set()
     log.info("語彙セットを構築中...")
@@ -125,7 +125,7 @@ def main():
     with open(input_file, encoding="utf-8") as f:
         try:
             project_dicts = json.load(f)
-            projects = [projects_schema.Project(**item) for item in project_dicts]
+            projects = [Project(**item) for item in project_dicts]
         except json.JSONDecodeError as e:
             log.error(f"JSONデコードエラー: {e}")
             sys.exit(1)

scripts/4_prepare_bm25f_meta.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import json
+import math
 from collections import defaultdict
 from typing import Dict, List
 
@@ -37,7 +38,9 @@ def build_tokenizer(sudachi_config_path: str):
     return tok, mode
 
 
-def tokenize(text: str, tok, mode, target_pos_l1: List[str], ban_list: set, stopwords: set) -> List[str]:
+def tokenize(
+    text: str, tok, mode, target_pos_l1: List[str], ban_list: set, stopwords: set
+) -> List[str]:
     if not text:
         return []
     out: List[str] = []
@@ -57,13 +60,19 @@ def tokenize(text: str, tok, mode, target_pos_l1: List[str], ban_list: set, stop
 def main():
     log.info("BM25Fメタデータ(bm25_meta.json)を生成します")
     try:
-        target_pos_l1, target_fields, ban_list, stopwords, sudachi_config_path = load_configs()
+        target_pos_l1, target_fields, ban_list, stopwords, sudachi_config_path = (
+            load_configs()
+        )
     except Exception as e:
         log.error(f"設定の読み込みに失敗しました: {e}")
         sys.exit(1)
 
-    projects_path = get_file_path_from_config("projects.projects_json", "data/generated/projects.json")
-    output_path = get_file_path_from_config("bm25.bm25_meta", "data/generated/bm25_meta.json")
+    projects_path = get_file_path_from_config(
+        "projects.projects_json", "data/generated/projects.json"
+    )
+    output_path = get_file_path_from_config(
+        "bm25.bm25_meta", "data/generated/bm25_meta.json"
+    )
 
     try:
         with open(projects_path, encoding="utf-8") as f:
@@ -94,13 +103,14 @@ def main():
     # IDF 計算（BM25で一般的な +0.5 smoothing と +1 オフセット）
     idf: Dict[str, float] = {}
     for term, dfi in df.items():
-        idf_val = max(0.0, ( ( (N - dfi + 0.5) / (dfi + 0.5) ) ))
+        idf_val = max(0.0, ((N - dfi + 0.5) / (dfi + 0.5)))
         # 数値安定化のためlog1p
-        import math
-
         idf[term] = math.log1p(idf_val)
 
-    avg_len = {field: (field_token_lens_sum[field] / N if N > 0 else 0.0) for field in target_fields}
+    avg_len = {
+        field: (field_token_lens_sum[field] / N if N > 0 else 0.0)
+        for field in target_fields
+    }
 
     meta = {
         "N": N,

scripts/5_prepare_tf_token.py

@@ -10,7 +10,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
 from utils.logger import setup_logger
 from utils.json import get_file_path_from_config, field_getter, json_dumps
-import schemas.tf_token as tf_schema
+from schemas import tf_token
 
 log = setup_logger(__name__)
 
@@ -38,7 +38,9 @@ def build_tokenizer(sudachi_config_path: str):
     return tok, mode
 
 
-def tokenize(text: str, tok, mode, target_pos_l1: List[str], ban_list: set, stopwords: set) -> List[str]:
+def tokenize(
+    text: str, tok, mode, target_pos_l1: List[str], ban_list: set, stopwords: set
+) -> List[str]:
     if not text:
         return []
     out: List[str] = []
@@ -58,13 +60,19 @@ def tokenize(text: str, tok, mode, target_pos_l1: List[str], ban_list: set, stop
 def main():
     log.info("フィールド別TF/トークン(tf_token.json)を生成します")
     try:
-        target_pos_l1, target_fields, ban_list, stopwords, sudachi_config_path = load_configs()
+        target_pos_l1, target_fields, ban_list, stopwords, sudachi_config_path = (
+            load_configs()
+        )
     except Exception as e:
         log.error(f"設定の読み込みに失敗しました: {e}")
         sys.exit(1)
 
-    projects_path = get_file_path_from_config("projects.projects_json", "data/generated/projects.json")
-    output_path = get_file_path_from_config("bm25.tf_token", "data/generated/tf_token.json")
+    projects_path = get_file_path_from_config(
+        "projects.projects_json", "data/generated/projects.json"
+    )
+    output_path = get_file_path_from_config(
+        "bm25.tf_token", "data/generated/tf_token.json"
+    )
 
     try:
         with open(projects_path, encoding="utf-8") as f:
@@ -75,13 +83,13 @@ def main():
 
     tok, mode = build_tokenizer(sudachi_config_path)
 
-    results: List[tf_schema.Project] = []
+    results: List[tf_token.Project] = []
 
     for p in projects:
         project_id = p.get("projectId")
 
         # 各フィールドのトークン化とTF
-        field_objs: Dict[str, tf_schema.TfOfField] = {}
+        field_objs: Dict[str, tf_token.TfOfField] = {}
         doc_tf_counter: Counter = Counter()
         doc_token_set: set = set()
 
@@ -89,15 +97,15 @@ def main():
             text = p.get(field) or ""
             toks = tokenize(str(text), tok, mode, target_pos_l1, ban_list, stopwords)
             tf = Counter(toks)
-            field_objs[field] = tf_schema.TfOfField(len=len(toks), tf=dict(tf))
+            field_objs[field] = tf_token.TfOfField(len=len(toks), tf=dict(tf))
             doc_tf_counter.update(tf)
             doc_token_set.update(tf.keys())
 
         # スキーマ Fields へ詰める（未定義フィールドは長さ0/空dictで埋める）
-        def get_field(name: str) -> tf_schema.TfOfField:
-            return field_objs.get(name, tf_schema.TfOfField(len=0, tf={}))
+        def get_field(name: str) -> tf_token.TfOfField:
+            return field_objs.get(name, tf_token.TfOfField(len=0, tf={}))
 
-        fields_obj = tf_schema.Fields(
+        fields_obj = tf_token.Fields(
             title=get_field("title"),
             organization=get_field("organization"),
             reading=get_field("reading"),
@@ -106,7 +114,7 @@ def main():
             prCommentLong=get_field("prCommentLong"),
         )
 
-        project_entry = tf_schema.Project(
+        project_entry = tf_token.Project(
             projectId=project_id,
             fields=fields_obj,
             tf=dict(doc_tf_counter),

scripts/7_prepare_circle_names.py

@@ -0,0 +1,103 @@
+import os
+import sys
+import json
+from collections import defaultdict
+from typing import Dict, List, Set
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from utils.logger import setup_logger
+from utils.json import get_file_path_from_config, json_dumps
+from schemas.projects import Project
+
+log = setup_logger(__name__)
+
+
+def normalized_circle_name(name: str) -> str:
+    """
+    サークル名の正規化を行う。
+    - 漢字・ひらがな・カタカナはそのまま
+    - 半角英数字は小文字に変換
+    - 記号類（！"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~・／？＜＞（）【】？｜：）を除く
+    """
+    SYMBOLS = set("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~・／？＜＞（）【】？｜：")
+    if not name:
+        return ""
+    normalized = []
+    for char in name:
+        if "A" <= char <= "Z":
+            normalized.append(char.lower())
+        elif char in SYMBOLS:
+            continue
+        else:
+            normalized.append(char)
+    return "".join(normalized)
+
+
+def get_substrings(texts: Dict[any, str]) -> Set[str]:
+    """テキストから2文字以上の連続部分文字列をすべて抽出する。"""
+    substrings = set()
+    for key, text in texts.items():
+        if key == "projectId":
+            continue
+        text = text.replace(" ", "").replace("　", "")
+        length = len(text)
+        for start in range(length):
+            for end in range(start + 2, length + 1):
+                substr = text[start:end]
+                substrings.add(substr)
+    return substrings
+
+
+def main():
+    input_file = get_file_path_from_config("projects.projects_json")
+
+    try:
+        with open(input_file, encoding="utf-8") as f:
+            projects = json.load(f)
+    except FileNotFoundError:
+        log.error(f"入力ファイルが見つかりません: {input_file}")
+        sys.exit(1)
+    except json.JSONDecodeError as e:
+        log.error(f"JSONデコードエラー: {e}")
+        sys.exit(1)
+
+    # --- 団体名データを生成 ---
+    output_file_names = get_file_path_from_config("substring.circle_names")
+    projects = [Project(**item) for item in projects]
+    circle_names = [
+        {
+            "projectId": p.projectId,
+            "circle": p.organization,
+            "circleNormalized": normalized_circle_name(p.organization),
+            "circleKana": p.reading,
+        }
+        for p in projects
+    ]
+
+    # 出力先ディレクトリ作成（念のため）
+    os.makedirs(os.path.dirname(output_file_names), exist_ok=True)
+
+    # JSON に書き出し
+    json_dumps(circle_names, output_file_names)
+    log.info(f"サークル名データを出力しました: {output_file_names}")
+
+    # --- 部分文字列インデックスを生成 ---
+    output_file_substring = get_file_path_from_config("substring.substring_index")
+    substring_to_project_ids: Dict[str, List[str]] = defaultdict(list)
+    for proj in circle_names:
+        projectId = proj["projectId"]
+        substrings = get_substrings(proj)
+        for substr in substrings:
+            substring_to_project_ids[substr].append(projectId)
+
+    # 出力先ディレクトリ作成（念のため）
+    os.makedirs(os.path.dirname(output_file_substring), exist_ok=True)
+
+    # JSON に書き出し
+    json_dumps(substring_to_project_ids, output_file_substring)
+    log.info(f"部分文字列インデックスを出力しました: {output_file_substring}")
+
+
+if __name__ == "__main__":
+    main()

scripts/build_all.py

@@ -51,6 +51,10 @@ def main():
     else:
         log.info(".vec が見つからないため Step 6 をスキップします: %s", vec_path)
 
+    # Step 7: substring index
+    run_step(
+        [sys.executable, "scripts/7_build_substring_index.py"]
+    )  # needs projects.json
     log.info("全ステップ完了")