ktsn-ud commited on
Commit
a089e74
·
1 Parent(s): f446d43

codex生成: API部分

Browse files
api/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ __all__ = []
2
+
api/main.py ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import gzip
4
+ from typing import List, Optional, Literal
5
+
6
+ from fastapi import FastAPI, Response, Depends, HTTPException, Security, Query
7
+ from fastapi.responses import RedirectResponse
8
+ from fastapi.security import APIKeyHeader
9
+ from pydantic import BaseModel
10
+
11
+ from utils.logger import setup_logger
12
+ from utils.json import field_getter
13
+ from api.search.engine import SearchEngine
14
+ import schemas.projects as schema_projects
15
+
16
+ log = setup_logger(__name__)
17
+
18
+
19
+ # --- Auth ---
20
+ API_KEY_NAME = "X-API-KEY"
21
+ api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=True)
22
+
23
+
24
+ def get_api_key(key: str = Security(api_key_header)):
25
+ secret = os.getenv("API_SECRET_KEY")
26
+ if not secret or key != secret:
27
+ raise HTTPException(status_code=403, detail="Could not validate credentials.")
28
+ return key
29
+
30
+
31
+ # --- App ---
32
+ engine = SearchEngine()
33
+ app = FastAPI()
34
+
35
+
36
+ @app.on_event("startup")
37
+ def on_startup():
38
+ log.info("Initializing search engine and loading assets...")
39
+ engine.initialize()
40
+ log.info("Initialization complete.")
41
+
42
+
43
+ @app.get("/", include_in_schema=False)
44
+ def root():
45
+ return RedirectResponse("/docs")
46
+
47
+
48
+ @app.get("/api/health")
49
+ def health_check():
50
+ return {"status": "ok"}
51
+
52
+
53
+ @app.get(
54
+ "/api/projects",
55
+ response_model=List[schema_projects.ProjectSummary],
56
+ dependencies=[Depends(get_api_key)],
57
+ )
58
+ def get_summary_data():
59
+ summary_fields = set(schema_projects.ProjectSummary.model_fields.keys())
60
+ summaries = [
61
+ {k: v for k, v in p.items() if k in summary_fields}
62
+ for p in engine.get_projects()
63
+ ]
64
+ content = json.dumps(summaries, ensure_ascii=False).encode("utf-8")
65
+ return Response(
66
+ content=gzip.compress(content),
67
+ headers={"Content-Encoding": "gzip", "Content-Type": "application/json"},
68
+ )
69
+
70
+
71
+ @app.get(
72
+ "/api/details",
73
+ response_model=schema_projects.ProjectDetail,
74
+ dependencies=[Depends(get_api_key)],
75
+ )
76
+ def get_project_detail(projectId: str = Query(..., description="取得したい企画のID")):
77
+ p = engine.get_project_map().get(projectId)
78
+ if not p:
79
+ raise HTTPException(status_code=404, detail="Project not found")
80
+ fields = set(schema_projects.ProjectDetail.model_fields.keys())
81
+ return {k: v for k, v in p.items() if k in fields}
82
+
83
+
84
+ class SearchRequest(BaseModel):
85
+ query: str
86
+ debug: bool = False
87
+
88
+
89
+ @app.post(
90
+ "/api/search",
91
+ response_model=schema_projects.ProjectIds,
92
+ dependencies=[Depends(get_api_key)],
93
+ )
94
+ def search(request: SearchRequest):
95
+ if not request.query:
96
+ raise HTTPException(status_code=400, detail="Query cannot be empty")
97
+
98
+ pairs = engine.search(request.query, debug=request.debug)
99
+ ids = [pid for pid, _ in pairs]
100
+ return schema_projects.ProjectIds(projectIds=ids)
api/search/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ __all__ = []
2
+
api/search/engine.py ADDED
@@ -0,0 +1,361 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ from dataclasses import dataclass
4
+ from typing import Any, Dict, List, Optional, Tuple
5
+
6
+ import numpy as np
7
+ from sudachipy import dictionary, tokenizer
8
+
9
+ from utils.json import field_getter
10
+ from utils.logger import setup_logger
11
+
12
+ log = setup_logger(__name__)
13
+
14
+
15
+ @dataclass
16
+ class SearchConfig:
17
+ target_pos_l1: List[str]
18
+ target_fields: List[str]
19
+ k1: float
20
+ b: float
21
+ field_weights: Dict[str, float]
22
+ synonyms_enable: bool
23
+ syn_limits: Dict[str, int]
24
+ banlist: List[str]
25
+ word_sim_enable: bool
26
+ word_sim_mode: str
27
+ word_sim_alpha: float
28
+ query_subword_enable: bool
29
+ query_subword_path: str
30
+ query_subword_oov_weight: float
31
+ org_boost_exact: float
32
+ org_boost_prefix: float
33
+ org_boost_substring: float
34
+ org_boost_min_len: int
35
+
36
+
37
+ def normalize_text_for_org(s: str) -> str:
38
+ try:
39
+ import unicodedata
40
+
41
+ s = unicodedata.normalize("NFKC", s)
42
+ except Exception:
43
+ pass
44
+ s = " ".join(s.split())
45
+ return s
46
+
47
+
48
+ class SearchEngine:
49
+ def __init__(self):
50
+ self.cfg: Optional[SearchConfig] = None
51
+ self.tokenizer = None
52
+ self.mode = None
53
+ self.stopwords: set[str] = set()
54
+ self.custom_synonyms: Dict[str, List[str]] = {}
55
+ self.synonyms_cache: Dict[str, List[str]] = {}
56
+
57
+ # Data
58
+ self.projects: List[Dict[str, Any]] = []
59
+ self.project_map: Dict[str, Dict[str, Any]] = {}
60
+ self.org_norms: Dict[str, str] = {}
61
+ self.reading_norms: Dict[str, str] = {}
62
+
63
+ # BM25F assets
64
+ self.idf: Dict[str, float] = {}
65
+ self.avg_len: Dict[str, float] = {}
66
+ self.tf_token_docs: List[Dict[str, Any]] = []
67
+
68
+ # Vectors
69
+ self.word_vocab: Dict[str, int] = {}
70
+ self.word_vectors: Optional[np.ndarray] = None
71
+ self.doc_vectors: Optional[np.ndarray] = None
72
+ self.ft_model = None
73
+
74
+ # ----- Init / Load -----
75
+ def initialize(self):
76
+ files = field_getter("config/files.json")
77
+ search = field_getter("config/search_model.json")
78
+
79
+ # Config
80
+ self.cfg = SearchConfig(
81
+ target_pos_l1=search("target_pos_l1"),
82
+ target_fields=search("target_fields"),
83
+ k1=float(search("bm25f.k1")),
84
+ b=float(search("bm25f.b")),
85
+ field_weights=search("bm25f.field_weights"),
86
+ synonyms_enable=bool(search("synonyms.enable")),
87
+ syn_limits=search("synonyms.limits"),
88
+ banlist=search("synonyms.banlist"),
89
+ word_sim_enable=bool(search("word_sim.enable")),
90
+ word_sim_mode=(search("word_sim.mode") or "soft").lower(),
91
+ word_sim_alpha=float(search("word_sim.alpha")),
92
+ query_subword_enable=bool(search("query_subword.enable")),
93
+ query_subword_path=search("query_subword.path"),
94
+ query_subword_oov_weight=float(search("query_subword.oov_weight")),
95
+ org_boost_exact=float(search("organization.boost.exact", 1.0)),
96
+ org_boost_prefix=float(search("organization.boost.prefix", 0.7)),
97
+ org_boost_substring=float(search("organization.boost.substring", 0.5)),
98
+ org_boost_min_len=int(search("organization.boost.min_len", 2)),
99
+ )
100
+
101
+ # Tokenizer
102
+ sudachi_config_path = files("sudachi.sudachi_config")
103
+ tok = dictionary.Dictionary(config_path=sudachi_config_path).create()
104
+ self.tokenizer = tok
105
+ self.mode = tokenizer.Tokenizer.SplitMode.A
106
+
107
+ # Stopwords
108
+ with open(files("sudachi.stopwords"), encoding="utf-8") as f:
109
+ self.stopwords = set(json.load(f))
110
+
111
+ # Synonyms assets
112
+ try:
113
+ syn_cache_path = files("sudachi.synonyms_cache")
114
+ if os.path.exists(syn_cache_path):
115
+ with open(syn_cache_path, encoding="utf-8") as f:
116
+ self.synonyms_cache = json.load(f)
117
+ except Exception as e:
118
+ log.warning(f"failed to load synonyms_cache: {e}")
119
+
120
+ try:
121
+ custom_path = field_getter("config/search_model.json")("synonyms.sources.custom_json")
122
+ if custom_path and os.path.exists(custom_path):
123
+ with open(custom_path, encoding="utf-8") as f:
124
+ self.custom_synonyms = json.load(f)
125
+ except Exception:
126
+ pass
127
+
128
+ # Projects
129
+ with open(files("projects.projects_json"), encoding="utf-8") as f:
130
+ self.projects = json.load(f)
131
+ self.project_map = {p["projectId"]: p for p in self.projects}
132
+ self.org_norms = {p["projectId"]: normalize_text_for_org(p.get("organization") or "") for p in self.projects}
133
+ self.reading_norms = {p["projectId"]: normalize_text_for_org(p.get("reading") or "") for p in self.projects}
134
+
135
+ # BM25F assets
136
+ with open(files("bm25.bm25_meta"), encoding="utf-8") as f:
137
+ meta = json.load(f)
138
+ self.idf = meta.get("idf", {})
139
+ self.avg_len = meta.get("avg_len", {})
140
+
141
+ with open(files("bm25.tf_token"), encoding="utf-8") as f:
142
+ self.tf_token_docs = json.load(f)
143
+
144
+ # Vectors
145
+ try:
146
+ vocab_path = files("embeddings.word_vocab")
147
+ vec_path = files("embeddings.word_vectors")
148
+ if os.path.exists(vocab_path) and os.path.exists(vec_path):
149
+ with open(vocab_path, encoding="utf-8") as f:
150
+ self.word_vocab = {k: int(v) for k, v in json.load(f).items()}
151
+ self.word_vectors = np.load(vec_path)["vectors"]
152
+ except Exception as e:
153
+ log.warning(f"word vectors not ready: {e}")
154
+
155
+ try:
156
+ doc_vec_path = files("embeddings.doc_vectors")
157
+ if os.path.exists(doc_vec_path):
158
+ self.doc_vectors = np.load(doc_vec_path)
159
+ except Exception as e:
160
+ log.warning(f"doc vectors not ready: {e}")
161
+
162
+ # fastText OOV
163
+ if self.cfg.query_subword_enable and self.cfg.query_subword_path and os.path.exists(self.cfg.query_subword_path):
164
+ try:
165
+ import fasttext
166
+
167
+ self.ft_model = fasttext.load_model(self.cfg.query_subword_path)
168
+ log.info("fastText .bin loaded for OOV")
169
+ except Exception as e:
170
+ log.warning(f"failed to load fastText .bin: {e}")
171
+
172
+ # ----- Tokenize / Synonyms -----
173
+ def _tokenize(self, text: str) -> List[str]:
174
+ if not text:
175
+ return []
176
+ out: List[str] = []
177
+ for m in self.tokenizer.tokenize(text, self.mode):
178
+ base = m.normalized_form().lower().strip()
179
+ if not base:
180
+ continue
181
+ pos = m.part_of_speech()
182
+ if pos[0] not in self.cfg.target_pos_l1:
183
+ continue
184
+ if base in self.stopwords or base in self.cfg.banlist:
185
+ continue
186
+ out.append(base)
187
+ return out
188
+
189
+ def _expand_synonyms(self, terms: List[str]) -> List[str]:
190
+ if not self.cfg.synonyms_enable:
191
+ return terms
192
+ max_exp = int(self.cfg.syn_limits.get("max_expansions_per_term", 4))
193
+ min_len = int(self.cfg.syn_limits.get("min_char_len", 2))
194
+ expanded: List[str] = []
195
+ for t in terms:
196
+ expanded.append(t)
197
+ cands = []
198
+ cands.extend(self.synonyms_cache.get(t, []))
199
+ cands.extend(self.custom_synonyms.get(t, []))
200
+ # filter/unique
201
+ uniq = []
202
+ seen = set()
203
+ for c in cands:
204
+ if c in seen or len(c) < min_len or c in self.cfg.banlist:
205
+ continue
206
+ seen.add(c)
207
+ uniq.append(c)
208
+ if len(uniq) >= max_exp:
209
+ break
210
+ expanded.extend(uniq)
211
+ # overall limit
212
+ max_q = int(self.cfg.syn_limits.get("max_query_variants", 5))
213
+ return expanded[: max_q * max_exp + len(terms)]
214
+
215
+ # ----- BM25F -----
216
+ def _bm25f_scores(self, terms: List[str]) -> np.ndarray:
217
+ N = len(self.tf_token_docs)
218
+ if N == 0:
219
+ return np.zeros((0,), dtype=np.float32)
220
+ k1 = self.cfg.k1
221
+ b = self.cfg.b
222
+ fw = self.cfg.field_weights
223
+
224
+ scores = np.zeros((N,), dtype=np.float32)
225
+
226
+ idf = self.idf
227
+ avg_len = self.avg_len
228
+
229
+ # For quick access, build list of per-doc per-field structures
230
+ for i, d in enumerate(self.tf_token_docs):
231
+ fields = d.get("fields") or {}
232
+ s = 0.0
233
+ for t in terms:
234
+ idf_t = float(idf.get(t, 0.0))
235
+ if idf_t <= 0.0:
236
+ continue
237
+ denom_sum = 0.0
238
+ num_sum = 0.0
239
+ for field, weight in fw.items():
240
+ fobj = (fields.get(field) or {})
241
+ tf = float((fobj.get("tf") or {}).get(t, 0))
242
+ if tf <= 0.0:
243
+ continue
244
+ len_f = float(fobj.get("len", 0))
245
+ avg_f = float(avg_len.get(field, 0.0)) or 1.0
246
+ norm = k1 * (1 - b + b * (len_f / avg_f))
247
+ num_sum += weight * tf * (k1 + 1.0)
248
+ denom_sum += weight * (tf + norm)
249
+ if denom_sum > 0:
250
+ s += idf_t * (num_sum / denom_sum)
251
+ scores[i] = s
252
+ return scores
253
+
254
+ # ----- Word similarity -----
255
+ def _get_token_vector(self, t: str) -> Tuple[Optional[np.ndarray], bool]:
256
+ if self.word_vectors is not None and t in self.word_vocab:
257
+ v = self.word_vectors[self.word_vocab[t]]
258
+ return v, False
259
+ if self.ft_model is not None:
260
+ try:
261
+ v = self.ft_model.get_word_vector(t)
262
+ v = v.astype(np.float32)
263
+ n = np.linalg.norm(v)
264
+ if n > 0:
265
+ v = v / n
266
+ return v, True
267
+ except Exception:
268
+ return None, True
269
+ return None, True
270
+
271
+ def _word_sim_scores(self, terms: List[str]) -> Optional[np.ndarray]:
272
+ if not self.cfg.word_sim_enable:
273
+ return None
274
+ if self.word_vectors is None or (self.cfg.word_sim_mode == "avg" and self.doc_vectors is None):
275
+ # avg needs doc_vectors; soft not implemented with per-term-per-doc words
276
+ return None
277
+
278
+ # Build query vector as IDF-weighted average; OOV vectors are down-weighted
279
+ weights = []
280
+ vecs = []
281
+ for t in terms:
282
+ v, oov = self._get_token_vector(t)
283
+ if v is None:
284
+ continue
285
+ w = float(self.idf.get(t, 0.0))
286
+ if oov:
287
+ w *= float(self.cfg.query_subword_oov_weight)
288
+ if w <= 0:
289
+ continue
290
+ vecs.append(v)
291
+ weights.append(w)
292
+
293
+ if not vecs:
294
+ return None
295
+
296
+ q = np.average(np.stack(vecs), axis=0, weights=np.asarray(weights, dtype=np.float32))
297
+ n = np.linalg.norm(q)
298
+ if n > 0:
299
+ q = q / n
300
+
301
+ # cosine with doc vectors
302
+ sims = self.doc_vectors @ q.astype(np.float32)
303
+ return sims
304
+
305
+ # ----- Public API -----
306
+ def search(
307
+ self,
308
+ query: str,
309
+ debug: bool = False,
310
+ ) -> List[Tuple[str, float]]:
311
+ terms = self._tokenize(query)
312
+ if self.cfg.synonyms_enable:
313
+ terms = self._expand_synonyms(terms)
314
+
315
+ # BM25F
316
+ bm25 = self._bm25f_scores(terms)
317
+ score = bm25.copy()
318
+
319
+ # word sim
320
+ ws = self._word_sim_scores(terms)
321
+ if ws is not None:
322
+ a = float(self.cfg.word_sim_alpha)
323
+ score = a * score + (1.0 - a) * ws
324
+
325
+ # Organization/reading auto-boost based on raw query substring match
326
+ qn = normalize_text_for_org(query)
327
+ if len(qn) >= int(self.cfg.org_boost_min_len):
328
+ exact = np.zeros((len(self.projects),), dtype=bool)
329
+ prefix = np.zeros_like(exact)
330
+ substr = np.zeros_like(exact)
331
+ for i, d in enumerate(self.projects):
332
+ pid = d.get("projectId")
333
+ on = self.org_norms.get(pid, "")
334
+ rn = self.reading_norms.get(pid, "")
335
+ if qn and (qn == on or (rn and qn == rn)):
336
+ exact[i] = True
337
+ elif qn and (on.startswith(qn) or (rn and rn.startswith(qn))):
338
+ prefix[i] = True
339
+ elif qn and ((qn in on) or (rn and qn in rn)):
340
+ substr[i] = True
341
+
342
+ boost = (
343
+ exact.astype(np.float32) * float(self.cfg.org_boost_exact)
344
+ + prefix.astype(np.float32) * float(self.cfg.org_boost_prefix)
345
+ + substr.astype(np.float32) * float(self.cfg.org_boost_substring)
346
+ )
347
+ score = score + boost
348
+
349
+ # collect results
350
+ ids = [d.get("projectId") for d in self.projects]
351
+ pairs = list(zip(ids, score.tolist()))
352
+
353
+ # sort
354
+ pairs.sort(key=lambda x: (-x[1], x[0]))
355
+ return pairs
356
+
357
+ def get_projects(self) -> List[Dict[str, Any]]:
358
+ return self.projects
359
+
360
+ def get_project_map(self) -> Dict[str, Dict[str, Any]]:
361
+ return self.project_map
config/search_model.json CHANGED
@@ -43,7 +43,7 @@
43
  }
44
  },
45
  "word_sim": {
46
- "enable": false,
47
  "mode": "soft",
48
  "alpha": 0.7
49
  },
@@ -52,5 +52,13 @@
52
  "path": "resources/embeddings/cc.ja.300.bin",
53
  "oov_weight": 0.8,
54
  "cache_size": 50000
 
 
 
 
 
 
 
 
55
  }
56
  }
 
43
  }
44
  },
45
  "word_sim": {
46
+ "enable": true,
47
  "mode": "soft",
48
  "alpha": 0.7
49
  },
 
52
  "path": "resources/embeddings/cc.ja.300.bin",
53
  "oov_weight": 0.8,
54
  "cache_size": 50000
55
+ },
56
+ "organization": {
57
+ "boost": {
58
+ "exact": 1.0,
59
+ "prefix": 0.7,
60
+ "substring": 0.5,
61
+ "min_len": 2
62
+ }
63
  }
64
  }
docs/old_index.md ADDED
@@ -0,0 +1,245 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ```py
2
+ import os
3
+ import sys
4
+ import json
5
+ import gzip
6
+ import logging
7
+ from dotenv import load_dotenv
8
+ from fastapi import FastAPI, Response, Depends, HTTPException, Security, Query
9
+ from fastapi.responses import RedirectResponse
10
+ from fastapi.security import APIKeyHeader
11
+ from pydantic import BaseModel
12
+ from sudachipy import dictionary
13
+ from dataclasses import asdict, fields
14
+ from typing import Optional, Literal, List, Dict
15
+ from pathlib import Path
16
+
17
+ # デバッグ用
18
+ from contextlib import asynccontextmanager
19
+ from fastapi.routing import APIRoute
20
+
21
+ # ローカルモジュールのインポート
22
+ from api import search_preprocess
23
+ from api import data_fetch
24
+ from api.search import pipeline as search_pipeline
25
+ from api.model import Project, ProjectSummary, ProjectDetail, ProjectIds
26
+
27
+ # 親ディレクトリをパスに追加して設定ファイルをインポート
28
+ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
29
+ from scripts.config import cfg_file, cfg_search_model, cfg_search_params
30
+
31
+ # --- ロギング設定 ---
32
+ _level_name = os.getenv("LOG_LEVEL", "INFO").upper()
33
+ _level = getattr(logging, _level_name, logging.INFO)
34
+ logging.basicConfig(level=_level, format="%(asctime)s [%(levelname)s]: %(message)s")
35
+ log = logging.getLogger(__name__)
36
+ log.setLevel(_level)
37
+
38
+
39
+ # --- グローバル変数 ---
40
+ # このファイルの場所を基準にプロジェクトルートを特定
41
+ PROJECT_ROOT = Path(__file__).resolve().parent.parent
42
+
43
+ # 設定情報
44
+ settings_file = cfg_file()
45
+ settings_search_model = cfg_search_model()
46
+ settings_search_params = cfg_search_params()
47
+
48
+ # モデルとデータ
49
+ tokenizer_obj = None
50
+ sentence_model = None # kept for compatibility; no longer used
51
+ all_projects: List[Project] = [] # populated via search_pipeline
52
+ project_map: Dict[str, Project] = {}
53
+ docs_for_search: Dict[str, Dict] = {}
54
+ bm25_meta: Dict = {}
55
+ synonyms_cache: Dict = {}
56
+ custom_synonyms: Dict = {}
57
+
58
+
59
+ # --- 初期化処理 ---
60
+
61
+
62
+ def initialize_sudachi_tokenizer() -> dictionary.Dictionary:
63
+ """SudachiPyトークナイザをユーザー辞書と共に初期化する"""
64
+ sudachi_config_path = PROJECT_ROOT / "scripts/sudachi.json"
65
+ try:
66
+ if sudachi_config_path.exists():
67
+ log.info(
68
+ "SudachiPy設定ファイルが見つかりました。ユーザー辞書で初期化します。"
69
+ )
70
+ with open(sudachi_config_path, "r", encoding="utf-8") as f:
71
+ config = json.load(f)
72
+
73
+ # userDictのパスを絶対パスに変換
74
+ if "userDict" in config:
75
+ config["userDict"] = [str(PROJECT_ROOT / p) for p in config["userDict"]]
76
+
77
+ # configオブジェクトをJSON文字列に変換して渡す
78
+ return dictionary.Dictionary(config=json.dumps(config)).create()
79
+ else:
80
+ log.info(
81
+ "SudachiPy設定ファイルが見つかりません。システム辞書のみ使用します。"
82
+ )
83
+ return dictionary.Dictionary().create()
84
+ except Exception as e:
85
+ log.error(f"SudachiPyの初期化に失敗しました: {e}")
86
+ raise
87
+
88
+
89
+ def load_all_data():
90
+ """Deprecated: Data is now loaded by api.search.pipeline.initialize()."""
91
+ log.info("load_all_data() is deprecated; using search_pipeline.initialize().")
92
+
93
+
94
+ @asynccontextmanager
95
+ async def lifespan(app: FastAPI):
96
+ """アプリケーションの起動時と終了時に実行される処理"""
97
+ # --- 起動時処理 ---
98
+ global tokenizer_obj, sentence_model, all_projects, project_map
99
+
100
+ log.info("アプリケーションを起動します...")
101
+
102
+ # 環境変数をロード
103
+ load_dotenv()
104
+
105
+ # トークナイザの初期化
106
+ tokenizer_obj = initialize_sudachi_tokenizer()
107
+ search_preprocess.set_tokenizer(tokenizer_obj)
108
+
109
+ # 生成物の確保(ローカル or HF Datasets からフェッチ)
110
+ try:
111
+ fetch_summary = data_fetch.orchestrate_fetch_and_validate(settings_search_model)
112
+ missing = fetch_summary.get("missing_before", [])
113
+ fetched = fetch_summary.get("fetched", {})
114
+ checks = fetch_summary.get("checks", {})
115
+ ok_list = [k for k, v in checks.items() if v]
116
+ log.info(
117
+ f"DATA PREP: missing_before={missing}, fetched={list(k for k,v in fetched.items() if v)}, checks_ok={ok_list}"
118
+ )
119
+ ws_conf = settings_search_model.get("word_sim", {}) or {}
120
+ ws_enabled = bool(ws_conf.get("enable"))
121
+ ws_mode = (ws_conf.get("mode") or "avg").lower()
122
+ ws_ready = (
123
+ checks.get("word_vectors.npz", False)
124
+ and checks.get("word_vocab.json", False)
125
+ and (True if ws_mode != "avg" else checks.get("doc_vectors.npy", False))
126
+ )
127
+ if ws_enabled:
128
+ log.info(f"WORD_SIM: requested mode={ws_mode}, assets_ready={ws_ready}")
129
+ except Exception as e:
130
+ log.warning(f"Data fetch/validate step failed: {e}")
131
+
132
+ # 検索パイプラインの初期化���データロード含む)
133
+ search_pipeline.initialize(tokenizer_obj)
134
+ # 既存エンドポイント互換のためローカル参照も持つ
135
+ all_projects = search_pipeline.get_projects()
136
+ project_map = search_pipeline.get_project_map()
137
+
138
+ # ルート情報のログ出力
139
+ for r in app.routes:
140
+ if isinstance(r, APIRoute):
141
+ log.info(f"ROUTE {list(r.methods)} {r.path}")
142
+ log.info(f"DOCS={app.docs_url} OPENAPI={app.openapi_url} REDOC={app.redoc_url}")
143
+ log.info("アプリケーションの準備が整いました。")
144
+
145
+ yield
146
+
147
+ # --- 終了時処理 ---
148
+ log.info("アプリケーションをシャットダウンします。")
149
+
150
+
151
+ # --- FastAPIアプリケーション設定 ---
152
+ app = FastAPI(lifespan=lifespan)
153
+
154
+ # .envファイルから環境変数を読み込む
155
+ load_dotenv()
156
+
157
+
158
+ # --- 認証設定 ---
159
+ API_KEY_NAME = "X-API-KEY"
160
+ API_SECRET_KEY = os.getenv("API_SECRET_KEY")
161
+ api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=True)
162
+
163
+
164
+ async def get_api_key(key: str = Security(api_key_header)):
165
+ """APIキーを検証する依存関係"""
166
+ if not API_SECRET_KEY or key != API_SECRET_KEY:
167
+ raise HTTPException(status_code=403, detail="Could not validate credentials.")
168
+ return key
169
+
170
+
171
+ # --- APIエンドポイント ---
172
+
173
+
174
+ @app.get("/", include_in_schema=False)
175
+ def root():
176
+ """ルートURLへのアクセスはドキュメントへリダイレクト"""
177
+ return RedirectResponse("/docs")
178
+
179
+
180
+ @app.get("/api/health")
181
+ def health_check():
182
+ """ヘルスチェック用エンドポイント"""
183
+ return {"status": "ok"}
184
+
185
+
186
+ @app.get(
187
+ "/api/projects",
188
+ response_model=List[ProjectSummary],
189
+ dependencies=[Depends(get_api_key)],
190
+ )
191
+ def get_summary_data():
192
+ """全企画のサマリー情報をGZIP圧縮して返す"""
193
+ summary_fields = {f.name for f in fields(ProjectSummary)}
194
+ summaries = [
195
+ {k: v for k, v in asdict(p).items() if k in summary_fields}
196
+ for p in search_pipeline.get_projects()
197
+ ]
198
+
199
+ content = json.dumps(summaries, ensure_ascii=False).encode("utf-8")
200
+ return Response(
201
+ content=gzip.compress(content),
202
+ headers={"Content-Encoding": "gzip", "Content-Type": "application/json"},
203
+ )
204
+
205
+
206
+ @app.get(
207
+ "/api/details",
208
+ response_model=ProjectDetail,
209
+ dependencies=[Depends(get_api_key)],
210
+ )
211
+ def get_project_detail(projectId: str = Query(..., description="取得したい企画のID")):
212
+ """指定されたIDの企画詳細情報を返す"""
213
+ project = search_pipeline.get_project_map().get(projectId)
214
+ if not project:
215
+ raise HTTPException(status_code=404, detail="Project not found")
216
+
217
+ detail_fields = {f.name for f in fields(ProjectDetail)}
218
+ project_dict = asdict(project)
219
+ return {key: project_dict[key] for key in detail_fields if key in project_dict}
220
+
221
+
222
+ # --- 検索エンドポイント ---
223
+
224
+
225
+ class SearchRequest(BaseModel):
226
+ query: str
227
+ # fusion は非推奨: 設定ファイルで制御し、ここでは受け取っても無視する
228
+ fusion: Optional[Literal["add", "mul"]] = None
229
+ debug: bool = False
230
+
231
+
232
+ @app.post(
233
+ "/api/search",
234
+ response_model=ProjectIds,
235
+ dependencies=[Depends(get_api_key)],
236
+ )
237
+ def search(request: SearchRequest) -> ProjectIds:
238
+ """BM25F中心の新パイプラインで検索し、企画IDを返す。"""
239
+ if not request.query:
240
+ raise HTTPException(status_code=400, detail="Query cannot be empty")
241
+ result_ids = search_pipeline.search(request.query, request.debug)
242
+ log.info(f"検索完了: {len(result_ids)} 件を返却")
243
+ return ProjectIds(projectIds=result_ids)
244
+
245
+ ```