Spaces:
Sleeping
Sleeping
ktsn-ud commited on
Commit ·
a089e74
1
Parent(s): f446d43
codex生成: API部分
Browse files- api/__init__.py +2 -0
- api/main.py +100 -0
- api/search/__init__.py +2 -0
- api/search/engine.py +361 -0
- config/search_model.json +9 -1
- docs/old_index.md +245 -0
api/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__all__ = []
|
| 2 |
+
|
api/main.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import json
|
| 3 |
+
import gzip
|
| 4 |
+
from typing import List, Optional, Literal
|
| 5 |
+
|
| 6 |
+
from fastapi import FastAPI, Response, Depends, HTTPException, Security, Query
|
| 7 |
+
from fastapi.responses import RedirectResponse
|
| 8 |
+
from fastapi.security import APIKeyHeader
|
| 9 |
+
from pydantic import BaseModel
|
| 10 |
+
|
| 11 |
+
from utils.logger import setup_logger
|
| 12 |
+
from utils.json import field_getter
|
| 13 |
+
from api.search.engine import SearchEngine
|
| 14 |
+
import schemas.projects as schema_projects
|
| 15 |
+
|
| 16 |
+
log = setup_logger(__name__)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
# --- Auth ---
|
| 20 |
+
API_KEY_NAME = "X-API-KEY"
|
| 21 |
+
api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=True)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def get_api_key(key: str = Security(api_key_header)):
|
| 25 |
+
secret = os.getenv("API_SECRET_KEY")
|
| 26 |
+
if not secret or key != secret:
|
| 27 |
+
raise HTTPException(status_code=403, detail="Could not validate credentials.")
|
| 28 |
+
return key
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
# --- App ---
|
| 32 |
+
engine = SearchEngine()
|
| 33 |
+
app = FastAPI()
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
@app.on_event("startup")
|
| 37 |
+
def on_startup():
|
| 38 |
+
log.info("Initializing search engine and loading assets...")
|
| 39 |
+
engine.initialize()
|
| 40 |
+
log.info("Initialization complete.")
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
@app.get("/", include_in_schema=False)
|
| 44 |
+
def root():
|
| 45 |
+
return RedirectResponse("/docs")
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
@app.get("/api/health")
|
| 49 |
+
def health_check():
|
| 50 |
+
return {"status": "ok"}
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@app.get(
|
| 54 |
+
"/api/projects",
|
| 55 |
+
response_model=List[schema_projects.ProjectSummary],
|
| 56 |
+
dependencies=[Depends(get_api_key)],
|
| 57 |
+
)
|
| 58 |
+
def get_summary_data():
|
| 59 |
+
summary_fields = set(schema_projects.ProjectSummary.model_fields.keys())
|
| 60 |
+
summaries = [
|
| 61 |
+
{k: v for k, v in p.items() if k in summary_fields}
|
| 62 |
+
for p in engine.get_projects()
|
| 63 |
+
]
|
| 64 |
+
content = json.dumps(summaries, ensure_ascii=False).encode("utf-8")
|
| 65 |
+
return Response(
|
| 66 |
+
content=gzip.compress(content),
|
| 67 |
+
headers={"Content-Encoding": "gzip", "Content-Type": "application/json"},
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
@app.get(
|
| 72 |
+
"/api/details",
|
| 73 |
+
response_model=schema_projects.ProjectDetail,
|
| 74 |
+
dependencies=[Depends(get_api_key)],
|
| 75 |
+
)
|
| 76 |
+
def get_project_detail(projectId: str = Query(..., description="取得したい企画のID")):
|
| 77 |
+
p = engine.get_project_map().get(projectId)
|
| 78 |
+
if not p:
|
| 79 |
+
raise HTTPException(status_code=404, detail="Project not found")
|
| 80 |
+
fields = set(schema_projects.ProjectDetail.model_fields.keys())
|
| 81 |
+
return {k: v for k, v in p.items() if k in fields}
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class SearchRequest(BaseModel):
|
| 85 |
+
query: str
|
| 86 |
+
debug: bool = False
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
@app.post(
|
| 90 |
+
"/api/search",
|
| 91 |
+
response_model=schema_projects.ProjectIds,
|
| 92 |
+
dependencies=[Depends(get_api_key)],
|
| 93 |
+
)
|
| 94 |
+
def search(request: SearchRequest):
|
| 95 |
+
if not request.query:
|
| 96 |
+
raise HTTPException(status_code=400, detail="Query cannot be empty")
|
| 97 |
+
|
| 98 |
+
pairs = engine.search(request.query, debug=request.debug)
|
| 99 |
+
ids = [pid for pid, _ in pairs]
|
| 100 |
+
return schema_projects.ProjectIds(projectIds=ids)
|
api/search/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__all__ = []
|
| 2 |
+
|
api/search/engine.py
ADDED
|
@@ -0,0 +1,361 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import os
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import Any, Dict, List, Optional, Tuple
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
from sudachipy import dictionary, tokenizer
|
| 8 |
+
|
| 9 |
+
from utils.json import field_getter
|
| 10 |
+
from utils.logger import setup_logger
|
| 11 |
+
|
| 12 |
+
log = setup_logger(__name__)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@dataclass
|
| 16 |
+
class SearchConfig:
|
| 17 |
+
target_pos_l1: List[str]
|
| 18 |
+
target_fields: List[str]
|
| 19 |
+
k1: float
|
| 20 |
+
b: float
|
| 21 |
+
field_weights: Dict[str, float]
|
| 22 |
+
synonyms_enable: bool
|
| 23 |
+
syn_limits: Dict[str, int]
|
| 24 |
+
banlist: List[str]
|
| 25 |
+
word_sim_enable: bool
|
| 26 |
+
word_sim_mode: str
|
| 27 |
+
word_sim_alpha: float
|
| 28 |
+
query_subword_enable: bool
|
| 29 |
+
query_subword_path: str
|
| 30 |
+
query_subword_oov_weight: float
|
| 31 |
+
org_boost_exact: float
|
| 32 |
+
org_boost_prefix: float
|
| 33 |
+
org_boost_substring: float
|
| 34 |
+
org_boost_min_len: int
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def normalize_text_for_org(s: str) -> str:
|
| 38 |
+
try:
|
| 39 |
+
import unicodedata
|
| 40 |
+
|
| 41 |
+
s = unicodedata.normalize("NFKC", s)
|
| 42 |
+
except Exception:
|
| 43 |
+
pass
|
| 44 |
+
s = " ".join(s.split())
|
| 45 |
+
return s
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class SearchEngine:
|
| 49 |
+
def __init__(self):
|
| 50 |
+
self.cfg: Optional[SearchConfig] = None
|
| 51 |
+
self.tokenizer = None
|
| 52 |
+
self.mode = None
|
| 53 |
+
self.stopwords: set[str] = set()
|
| 54 |
+
self.custom_synonyms: Dict[str, List[str]] = {}
|
| 55 |
+
self.synonyms_cache: Dict[str, List[str]] = {}
|
| 56 |
+
|
| 57 |
+
# Data
|
| 58 |
+
self.projects: List[Dict[str, Any]] = []
|
| 59 |
+
self.project_map: Dict[str, Dict[str, Any]] = {}
|
| 60 |
+
self.org_norms: Dict[str, str] = {}
|
| 61 |
+
self.reading_norms: Dict[str, str] = {}
|
| 62 |
+
|
| 63 |
+
# BM25F assets
|
| 64 |
+
self.idf: Dict[str, float] = {}
|
| 65 |
+
self.avg_len: Dict[str, float] = {}
|
| 66 |
+
self.tf_token_docs: List[Dict[str, Any]] = []
|
| 67 |
+
|
| 68 |
+
# Vectors
|
| 69 |
+
self.word_vocab: Dict[str, int] = {}
|
| 70 |
+
self.word_vectors: Optional[np.ndarray] = None
|
| 71 |
+
self.doc_vectors: Optional[np.ndarray] = None
|
| 72 |
+
self.ft_model = None
|
| 73 |
+
|
| 74 |
+
# ----- Init / Load -----
|
| 75 |
+
def initialize(self):
|
| 76 |
+
files = field_getter("config/files.json")
|
| 77 |
+
search = field_getter("config/search_model.json")
|
| 78 |
+
|
| 79 |
+
# Config
|
| 80 |
+
self.cfg = SearchConfig(
|
| 81 |
+
target_pos_l1=search("target_pos_l1"),
|
| 82 |
+
target_fields=search("target_fields"),
|
| 83 |
+
k1=float(search("bm25f.k1")),
|
| 84 |
+
b=float(search("bm25f.b")),
|
| 85 |
+
field_weights=search("bm25f.field_weights"),
|
| 86 |
+
synonyms_enable=bool(search("synonyms.enable")),
|
| 87 |
+
syn_limits=search("synonyms.limits"),
|
| 88 |
+
banlist=search("synonyms.banlist"),
|
| 89 |
+
word_sim_enable=bool(search("word_sim.enable")),
|
| 90 |
+
word_sim_mode=(search("word_sim.mode") or "soft").lower(),
|
| 91 |
+
word_sim_alpha=float(search("word_sim.alpha")),
|
| 92 |
+
query_subword_enable=bool(search("query_subword.enable")),
|
| 93 |
+
query_subword_path=search("query_subword.path"),
|
| 94 |
+
query_subword_oov_weight=float(search("query_subword.oov_weight")),
|
| 95 |
+
org_boost_exact=float(search("organization.boost.exact", 1.0)),
|
| 96 |
+
org_boost_prefix=float(search("organization.boost.prefix", 0.7)),
|
| 97 |
+
org_boost_substring=float(search("organization.boost.substring", 0.5)),
|
| 98 |
+
org_boost_min_len=int(search("organization.boost.min_len", 2)),
|
| 99 |
+
)
|
| 100 |
+
|
| 101 |
+
# Tokenizer
|
| 102 |
+
sudachi_config_path = files("sudachi.sudachi_config")
|
| 103 |
+
tok = dictionary.Dictionary(config_path=sudachi_config_path).create()
|
| 104 |
+
self.tokenizer = tok
|
| 105 |
+
self.mode = tokenizer.Tokenizer.SplitMode.A
|
| 106 |
+
|
| 107 |
+
# Stopwords
|
| 108 |
+
with open(files("sudachi.stopwords"), encoding="utf-8") as f:
|
| 109 |
+
self.stopwords = set(json.load(f))
|
| 110 |
+
|
| 111 |
+
# Synonyms assets
|
| 112 |
+
try:
|
| 113 |
+
syn_cache_path = files("sudachi.synonyms_cache")
|
| 114 |
+
if os.path.exists(syn_cache_path):
|
| 115 |
+
with open(syn_cache_path, encoding="utf-8") as f:
|
| 116 |
+
self.synonyms_cache = json.load(f)
|
| 117 |
+
except Exception as e:
|
| 118 |
+
log.warning(f"failed to load synonyms_cache: {e}")
|
| 119 |
+
|
| 120 |
+
try:
|
| 121 |
+
custom_path = field_getter("config/search_model.json")("synonyms.sources.custom_json")
|
| 122 |
+
if custom_path and os.path.exists(custom_path):
|
| 123 |
+
with open(custom_path, encoding="utf-8") as f:
|
| 124 |
+
self.custom_synonyms = json.load(f)
|
| 125 |
+
except Exception:
|
| 126 |
+
pass
|
| 127 |
+
|
| 128 |
+
# Projects
|
| 129 |
+
with open(files("projects.projects_json"), encoding="utf-8") as f:
|
| 130 |
+
self.projects = json.load(f)
|
| 131 |
+
self.project_map = {p["projectId"]: p for p in self.projects}
|
| 132 |
+
self.org_norms = {p["projectId"]: normalize_text_for_org(p.get("organization") or "") for p in self.projects}
|
| 133 |
+
self.reading_norms = {p["projectId"]: normalize_text_for_org(p.get("reading") or "") for p in self.projects}
|
| 134 |
+
|
| 135 |
+
# BM25F assets
|
| 136 |
+
with open(files("bm25.bm25_meta"), encoding="utf-8") as f:
|
| 137 |
+
meta = json.load(f)
|
| 138 |
+
self.idf = meta.get("idf", {})
|
| 139 |
+
self.avg_len = meta.get("avg_len", {})
|
| 140 |
+
|
| 141 |
+
with open(files("bm25.tf_token"), encoding="utf-8") as f:
|
| 142 |
+
self.tf_token_docs = json.load(f)
|
| 143 |
+
|
| 144 |
+
# Vectors
|
| 145 |
+
try:
|
| 146 |
+
vocab_path = files("embeddings.word_vocab")
|
| 147 |
+
vec_path = files("embeddings.word_vectors")
|
| 148 |
+
if os.path.exists(vocab_path) and os.path.exists(vec_path):
|
| 149 |
+
with open(vocab_path, encoding="utf-8") as f:
|
| 150 |
+
self.word_vocab = {k: int(v) for k, v in json.load(f).items()}
|
| 151 |
+
self.word_vectors = np.load(vec_path)["vectors"]
|
| 152 |
+
except Exception as e:
|
| 153 |
+
log.warning(f"word vectors not ready: {e}")
|
| 154 |
+
|
| 155 |
+
try:
|
| 156 |
+
doc_vec_path = files("embeddings.doc_vectors")
|
| 157 |
+
if os.path.exists(doc_vec_path):
|
| 158 |
+
self.doc_vectors = np.load(doc_vec_path)
|
| 159 |
+
except Exception as e:
|
| 160 |
+
log.warning(f"doc vectors not ready: {e}")
|
| 161 |
+
|
| 162 |
+
# fastText OOV
|
| 163 |
+
if self.cfg.query_subword_enable and self.cfg.query_subword_path and os.path.exists(self.cfg.query_subword_path):
|
| 164 |
+
try:
|
| 165 |
+
import fasttext
|
| 166 |
+
|
| 167 |
+
self.ft_model = fasttext.load_model(self.cfg.query_subword_path)
|
| 168 |
+
log.info("fastText .bin loaded for OOV")
|
| 169 |
+
except Exception as e:
|
| 170 |
+
log.warning(f"failed to load fastText .bin: {e}")
|
| 171 |
+
|
| 172 |
+
# ----- Tokenize / Synonyms -----
|
| 173 |
+
def _tokenize(self, text: str) -> List[str]:
|
| 174 |
+
if not text:
|
| 175 |
+
return []
|
| 176 |
+
out: List[str] = []
|
| 177 |
+
for m in self.tokenizer.tokenize(text, self.mode):
|
| 178 |
+
base = m.normalized_form().lower().strip()
|
| 179 |
+
if not base:
|
| 180 |
+
continue
|
| 181 |
+
pos = m.part_of_speech()
|
| 182 |
+
if pos[0] not in self.cfg.target_pos_l1:
|
| 183 |
+
continue
|
| 184 |
+
if base in self.stopwords or base in self.cfg.banlist:
|
| 185 |
+
continue
|
| 186 |
+
out.append(base)
|
| 187 |
+
return out
|
| 188 |
+
|
| 189 |
+
def _expand_synonyms(self, terms: List[str]) -> List[str]:
|
| 190 |
+
if not self.cfg.synonyms_enable:
|
| 191 |
+
return terms
|
| 192 |
+
max_exp = int(self.cfg.syn_limits.get("max_expansions_per_term", 4))
|
| 193 |
+
min_len = int(self.cfg.syn_limits.get("min_char_len", 2))
|
| 194 |
+
expanded: List[str] = []
|
| 195 |
+
for t in terms:
|
| 196 |
+
expanded.append(t)
|
| 197 |
+
cands = []
|
| 198 |
+
cands.extend(self.synonyms_cache.get(t, []))
|
| 199 |
+
cands.extend(self.custom_synonyms.get(t, []))
|
| 200 |
+
# filter/unique
|
| 201 |
+
uniq = []
|
| 202 |
+
seen = set()
|
| 203 |
+
for c in cands:
|
| 204 |
+
if c in seen or len(c) < min_len or c in self.cfg.banlist:
|
| 205 |
+
continue
|
| 206 |
+
seen.add(c)
|
| 207 |
+
uniq.append(c)
|
| 208 |
+
if len(uniq) >= max_exp:
|
| 209 |
+
break
|
| 210 |
+
expanded.extend(uniq)
|
| 211 |
+
# overall limit
|
| 212 |
+
max_q = int(self.cfg.syn_limits.get("max_query_variants", 5))
|
| 213 |
+
return expanded[: max_q * max_exp + len(terms)]
|
| 214 |
+
|
| 215 |
+
# ----- BM25F -----
|
| 216 |
+
def _bm25f_scores(self, terms: List[str]) -> np.ndarray:
|
| 217 |
+
N = len(self.tf_token_docs)
|
| 218 |
+
if N == 0:
|
| 219 |
+
return np.zeros((0,), dtype=np.float32)
|
| 220 |
+
k1 = self.cfg.k1
|
| 221 |
+
b = self.cfg.b
|
| 222 |
+
fw = self.cfg.field_weights
|
| 223 |
+
|
| 224 |
+
scores = np.zeros((N,), dtype=np.float32)
|
| 225 |
+
|
| 226 |
+
idf = self.idf
|
| 227 |
+
avg_len = self.avg_len
|
| 228 |
+
|
| 229 |
+
# For quick access, build list of per-doc per-field structures
|
| 230 |
+
for i, d in enumerate(self.tf_token_docs):
|
| 231 |
+
fields = d.get("fields") or {}
|
| 232 |
+
s = 0.0
|
| 233 |
+
for t in terms:
|
| 234 |
+
idf_t = float(idf.get(t, 0.0))
|
| 235 |
+
if idf_t <= 0.0:
|
| 236 |
+
continue
|
| 237 |
+
denom_sum = 0.0
|
| 238 |
+
num_sum = 0.0
|
| 239 |
+
for field, weight in fw.items():
|
| 240 |
+
fobj = (fields.get(field) or {})
|
| 241 |
+
tf = float((fobj.get("tf") or {}).get(t, 0))
|
| 242 |
+
if tf <= 0.0:
|
| 243 |
+
continue
|
| 244 |
+
len_f = float(fobj.get("len", 0))
|
| 245 |
+
avg_f = float(avg_len.get(field, 0.0)) or 1.0
|
| 246 |
+
norm = k1 * (1 - b + b * (len_f / avg_f))
|
| 247 |
+
num_sum += weight * tf * (k1 + 1.0)
|
| 248 |
+
denom_sum += weight * (tf + norm)
|
| 249 |
+
if denom_sum > 0:
|
| 250 |
+
s += idf_t * (num_sum / denom_sum)
|
| 251 |
+
scores[i] = s
|
| 252 |
+
return scores
|
| 253 |
+
|
| 254 |
+
# ----- Word similarity -----
|
| 255 |
+
def _get_token_vector(self, t: str) -> Tuple[Optional[np.ndarray], bool]:
|
| 256 |
+
if self.word_vectors is not None and t in self.word_vocab:
|
| 257 |
+
v = self.word_vectors[self.word_vocab[t]]
|
| 258 |
+
return v, False
|
| 259 |
+
if self.ft_model is not None:
|
| 260 |
+
try:
|
| 261 |
+
v = self.ft_model.get_word_vector(t)
|
| 262 |
+
v = v.astype(np.float32)
|
| 263 |
+
n = np.linalg.norm(v)
|
| 264 |
+
if n > 0:
|
| 265 |
+
v = v / n
|
| 266 |
+
return v, True
|
| 267 |
+
except Exception:
|
| 268 |
+
return None, True
|
| 269 |
+
return None, True
|
| 270 |
+
|
| 271 |
+
def _word_sim_scores(self, terms: List[str]) -> Optional[np.ndarray]:
|
| 272 |
+
if not self.cfg.word_sim_enable:
|
| 273 |
+
return None
|
| 274 |
+
if self.word_vectors is None or (self.cfg.word_sim_mode == "avg" and self.doc_vectors is None):
|
| 275 |
+
# avg needs doc_vectors; soft not implemented with per-term-per-doc words
|
| 276 |
+
return None
|
| 277 |
+
|
| 278 |
+
# Build query vector as IDF-weighted average; OOV vectors are down-weighted
|
| 279 |
+
weights = []
|
| 280 |
+
vecs = []
|
| 281 |
+
for t in terms:
|
| 282 |
+
v, oov = self._get_token_vector(t)
|
| 283 |
+
if v is None:
|
| 284 |
+
continue
|
| 285 |
+
w = float(self.idf.get(t, 0.0))
|
| 286 |
+
if oov:
|
| 287 |
+
w *= float(self.cfg.query_subword_oov_weight)
|
| 288 |
+
if w <= 0:
|
| 289 |
+
continue
|
| 290 |
+
vecs.append(v)
|
| 291 |
+
weights.append(w)
|
| 292 |
+
|
| 293 |
+
if not vecs:
|
| 294 |
+
return None
|
| 295 |
+
|
| 296 |
+
q = np.average(np.stack(vecs), axis=0, weights=np.asarray(weights, dtype=np.float32))
|
| 297 |
+
n = np.linalg.norm(q)
|
| 298 |
+
if n > 0:
|
| 299 |
+
q = q / n
|
| 300 |
+
|
| 301 |
+
# cosine with doc vectors
|
| 302 |
+
sims = self.doc_vectors @ q.astype(np.float32)
|
| 303 |
+
return sims
|
| 304 |
+
|
| 305 |
+
# ----- Public API -----
|
| 306 |
+
def search(
|
| 307 |
+
self,
|
| 308 |
+
query: str,
|
| 309 |
+
debug: bool = False,
|
| 310 |
+
) -> List[Tuple[str, float]]:
|
| 311 |
+
terms = self._tokenize(query)
|
| 312 |
+
if self.cfg.synonyms_enable:
|
| 313 |
+
terms = self._expand_synonyms(terms)
|
| 314 |
+
|
| 315 |
+
# BM25F
|
| 316 |
+
bm25 = self._bm25f_scores(terms)
|
| 317 |
+
score = bm25.copy()
|
| 318 |
+
|
| 319 |
+
# word sim
|
| 320 |
+
ws = self._word_sim_scores(terms)
|
| 321 |
+
if ws is not None:
|
| 322 |
+
a = float(self.cfg.word_sim_alpha)
|
| 323 |
+
score = a * score + (1.0 - a) * ws
|
| 324 |
+
|
| 325 |
+
# Organization/reading auto-boost based on raw query substring match
|
| 326 |
+
qn = normalize_text_for_org(query)
|
| 327 |
+
if len(qn) >= int(self.cfg.org_boost_min_len):
|
| 328 |
+
exact = np.zeros((len(self.projects),), dtype=bool)
|
| 329 |
+
prefix = np.zeros_like(exact)
|
| 330 |
+
substr = np.zeros_like(exact)
|
| 331 |
+
for i, d in enumerate(self.projects):
|
| 332 |
+
pid = d.get("projectId")
|
| 333 |
+
on = self.org_norms.get(pid, "")
|
| 334 |
+
rn = self.reading_norms.get(pid, "")
|
| 335 |
+
if qn and (qn == on or (rn and qn == rn)):
|
| 336 |
+
exact[i] = True
|
| 337 |
+
elif qn and (on.startswith(qn) or (rn and rn.startswith(qn))):
|
| 338 |
+
prefix[i] = True
|
| 339 |
+
elif qn and ((qn in on) or (rn and qn in rn)):
|
| 340 |
+
substr[i] = True
|
| 341 |
+
|
| 342 |
+
boost = (
|
| 343 |
+
exact.astype(np.float32) * float(self.cfg.org_boost_exact)
|
| 344 |
+
+ prefix.astype(np.float32) * float(self.cfg.org_boost_prefix)
|
| 345 |
+
+ substr.astype(np.float32) * float(self.cfg.org_boost_substring)
|
| 346 |
+
)
|
| 347 |
+
score = score + boost
|
| 348 |
+
|
| 349 |
+
# collect results
|
| 350 |
+
ids = [d.get("projectId") for d in self.projects]
|
| 351 |
+
pairs = list(zip(ids, score.tolist()))
|
| 352 |
+
|
| 353 |
+
# sort
|
| 354 |
+
pairs.sort(key=lambda x: (-x[1], x[0]))
|
| 355 |
+
return pairs
|
| 356 |
+
|
| 357 |
+
def get_projects(self) -> List[Dict[str, Any]]:
|
| 358 |
+
return self.projects
|
| 359 |
+
|
| 360 |
+
def get_project_map(self) -> Dict[str, Dict[str, Any]]:
|
| 361 |
+
return self.project_map
|
config/search_model.json
CHANGED
|
@@ -43,7 +43,7 @@
|
|
| 43 |
}
|
| 44 |
},
|
| 45 |
"word_sim": {
|
| 46 |
-
"enable":
|
| 47 |
"mode": "soft",
|
| 48 |
"alpha": 0.7
|
| 49 |
},
|
|
@@ -52,5 +52,13 @@
|
|
| 52 |
"path": "resources/embeddings/cc.ja.300.bin",
|
| 53 |
"oov_weight": 0.8,
|
| 54 |
"cache_size": 50000
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
}
|
| 56 |
}
|
|
|
|
| 43 |
}
|
| 44 |
},
|
| 45 |
"word_sim": {
|
| 46 |
+
"enable": true,
|
| 47 |
"mode": "soft",
|
| 48 |
"alpha": 0.7
|
| 49 |
},
|
|
|
|
| 52 |
"path": "resources/embeddings/cc.ja.300.bin",
|
| 53 |
"oov_weight": 0.8,
|
| 54 |
"cache_size": 50000
|
| 55 |
+
},
|
| 56 |
+
"organization": {
|
| 57 |
+
"boost": {
|
| 58 |
+
"exact": 1.0,
|
| 59 |
+
"prefix": 0.7,
|
| 60 |
+
"substring": 0.5,
|
| 61 |
+
"min_len": 2
|
| 62 |
+
}
|
| 63 |
}
|
| 64 |
}
|
docs/old_index.md
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
```py
|
| 2 |
+
import os
|
| 3 |
+
import sys
|
| 4 |
+
import json
|
| 5 |
+
import gzip
|
| 6 |
+
import logging
|
| 7 |
+
from dotenv import load_dotenv
|
| 8 |
+
from fastapi import FastAPI, Response, Depends, HTTPException, Security, Query
|
| 9 |
+
from fastapi.responses import RedirectResponse
|
| 10 |
+
from fastapi.security import APIKeyHeader
|
| 11 |
+
from pydantic import BaseModel
|
| 12 |
+
from sudachipy import dictionary
|
| 13 |
+
from dataclasses import asdict, fields
|
| 14 |
+
from typing import Optional, Literal, List, Dict
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
|
| 17 |
+
# デバッグ用
|
| 18 |
+
from contextlib import asynccontextmanager
|
| 19 |
+
from fastapi.routing import APIRoute
|
| 20 |
+
|
| 21 |
+
# ローカルモジュールのインポート
|
| 22 |
+
from api import search_preprocess
|
| 23 |
+
from api import data_fetch
|
| 24 |
+
from api.search import pipeline as search_pipeline
|
| 25 |
+
from api.model import Project, ProjectSummary, ProjectDetail, ProjectIds
|
| 26 |
+
|
| 27 |
+
# 親ディレクトリをパスに追加して設定ファイルをインポート
|
| 28 |
+
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
| 29 |
+
from scripts.config import cfg_file, cfg_search_model, cfg_search_params
|
| 30 |
+
|
| 31 |
+
# --- ロギング設定 ---
|
| 32 |
+
_level_name = os.getenv("LOG_LEVEL", "INFO").upper()
|
| 33 |
+
_level = getattr(logging, _level_name, logging.INFO)
|
| 34 |
+
logging.basicConfig(level=_level, format="%(asctime)s [%(levelname)s]: %(message)s")
|
| 35 |
+
log = logging.getLogger(__name__)
|
| 36 |
+
log.setLevel(_level)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# --- グローバル変数 ---
|
| 40 |
+
# このファイルの場所を基準にプロジェクトルートを特定
|
| 41 |
+
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
| 42 |
+
|
| 43 |
+
# 設定情報
|
| 44 |
+
settings_file = cfg_file()
|
| 45 |
+
settings_search_model = cfg_search_model()
|
| 46 |
+
settings_search_params = cfg_search_params()
|
| 47 |
+
|
| 48 |
+
# モデルとデータ
|
| 49 |
+
tokenizer_obj = None
|
| 50 |
+
sentence_model = None # kept for compatibility; no longer used
|
| 51 |
+
all_projects: List[Project] = [] # populated via search_pipeline
|
| 52 |
+
project_map: Dict[str, Project] = {}
|
| 53 |
+
docs_for_search: Dict[str, Dict] = {}
|
| 54 |
+
bm25_meta: Dict = {}
|
| 55 |
+
synonyms_cache: Dict = {}
|
| 56 |
+
custom_synonyms: Dict = {}
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
# --- 初期化処理 ---
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def initialize_sudachi_tokenizer() -> dictionary.Dictionary:
|
| 63 |
+
"""SudachiPyトークナイザをユーザー辞書と共に初期化する"""
|
| 64 |
+
sudachi_config_path = PROJECT_ROOT / "scripts/sudachi.json"
|
| 65 |
+
try:
|
| 66 |
+
if sudachi_config_path.exists():
|
| 67 |
+
log.info(
|
| 68 |
+
"SudachiPy設定ファイルが見つかりました。ユーザー辞書で初期化します。"
|
| 69 |
+
)
|
| 70 |
+
with open(sudachi_config_path, "r", encoding="utf-8") as f:
|
| 71 |
+
config = json.load(f)
|
| 72 |
+
|
| 73 |
+
# userDictのパスを絶対パスに変換
|
| 74 |
+
if "userDict" in config:
|
| 75 |
+
config["userDict"] = [str(PROJECT_ROOT / p) for p in config["userDict"]]
|
| 76 |
+
|
| 77 |
+
# configオブジェクトをJSON文字列に変換して渡す
|
| 78 |
+
return dictionary.Dictionary(config=json.dumps(config)).create()
|
| 79 |
+
else:
|
| 80 |
+
log.info(
|
| 81 |
+
"SudachiPy設定ファイルが見つかりません。システム辞書のみ使用します。"
|
| 82 |
+
)
|
| 83 |
+
return dictionary.Dictionary().create()
|
| 84 |
+
except Exception as e:
|
| 85 |
+
log.error(f"SudachiPyの初期化に失敗しました: {e}")
|
| 86 |
+
raise
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def load_all_data():
|
| 90 |
+
"""Deprecated: Data is now loaded by api.search.pipeline.initialize()."""
|
| 91 |
+
log.info("load_all_data() is deprecated; using search_pipeline.initialize().")
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
@asynccontextmanager
|
| 95 |
+
async def lifespan(app: FastAPI):
|
| 96 |
+
"""アプリケーションの起動時と終了時に実行される処理"""
|
| 97 |
+
# --- 起動時処理 ---
|
| 98 |
+
global tokenizer_obj, sentence_model, all_projects, project_map
|
| 99 |
+
|
| 100 |
+
log.info("アプリケーションを起動します...")
|
| 101 |
+
|
| 102 |
+
# 環境変数をロード
|
| 103 |
+
load_dotenv()
|
| 104 |
+
|
| 105 |
+
# トークナイザの初期化
|
| 106 |
+
tokenizer_obj = initialize_sudachi_tokenizer()
|
| 107 |
+
search_preprocess.set_tokenizer(tokenizer_obj)
|
| 108 |
+
|
| 109 |
+
# 生成物の確保(ローカル or HF Datasets からフェッチ)
|
| 110 |
+
try:
|
| 111 |
+
fetch_summary = data_fetch.orchestrate_fetch_and_validate(settings_search_model)
|
| 112 |
+
missing = fetch_summary.get("missing_before", [])
|
| 113 |
+
fetched = fetch_summary.get("fetched", {})
|
| 114 |
+
checks = fetch_summary.get("checks", {})
|
| 115 |
+
ok_list = [k for k, v in checks.items() if v]
|
| 116 |
+
log.info(
|
| 117 |
+
f"DATA PREP: missing_before={missing}, fetched={list(k for k,v in fetched.items() if v)}, checks_ok={ok_list}"
|
| 118 |
+
)
|
| 119 |
+
ws_conf = settings_search_model.get("word_sim", {}) or {}
|
| 120 |
+
ws_enabled = bool(ws_conf.get("enable"))
|
| 121 |
+
ws_mode = (ws_conf.get("mode") or "avg").lower()
|
| 122 |
+
ws_ready = (
|
| 123 |
+
checks.get("word_vectors.npz", False)
|
| 124 |
+
and checks.get("word_vocab.json", False)
|
| 125 |
+
and (True if ws_mode != "avg" else checks.get("doc_vectors.npy", False))
|
| 126 |
+
)
|
| 127 |
+
if ws_enabled:
|
| 128 |
+
log.info(f"WORD_SIM: requested mode={ws_mode}, assets_ready={ws_ready}")
|
| 129 |
+
except Exception as e:
|
| 130 |
+
log.warning(f"Data fetch/validate step failed: {e}")
|
| 131 |
+
|
| 132 |
+
# 検索パイプラインの初期化���データロード含む)
|
| 133 |
+
search_pipeline.initialize(tokenizer_obj)
|
| 134 |
+
# 既存エンドポイント互換のためローカル参照も持つ
|
| 135 |
+
all_projects = search_pipeline.get_projects()
|
| 136 |
+
project_map = search_pipeline.get_project_map()
|
| 137 |
+
|
| 138 |
+
# ルート情報のログ出力
|
| 139 |
+
for r in app.routes:
|
| 140 |
+
if isinstance(r, APIRoute):
|
| 141 |
+
log.info(f"ROUTE {list(r.methods)} {r.path}")
|
| 142 |
+
log.info(f"DOCS={app.docs_url} OPENAPI={app.openapi_url} REDOC={app.redoc_url}")
|
| 143 |
+
log.info("アプリケーションの準備が整いました。")
|
| 144 |
+
|
| 145 |
+
yield
|
| 146 |
+
|
| 147 |
+
# --- 終了時処理 ---
|
| 148 |
+
log.info("アプリケーションをシャットダウンします。")
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
# --- FastAPIアプリケーション設定 ---
|
| 152 |
+
app = FastAPI(lifespan=lifespan)
|
| 153 |
+
|
| 154 |
+
# .envファイルから環境変数を読み込む
|
| 155 |
+
load_dotenv()
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
# --- 認証設定 ---
|
| 159 |
+
API_KEY_NAME = "X-API-KEY"
|
| 160 |
+
API_SECRET_KEY = os.getenv("API_SECRET_KEY")
|
| 161 |
+
api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=True)
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
async def get_api_key(key: str = Security(api_key_header)):
|
| 165 |
+
"""APIキーを検証する依存関係"""
|
| 166 |
+
if not API_SECRET_KEY or key != API_SECRET_KEY:
|
| 167 |
+
raise HTTPException(status_code=403, detail="Could not validate credentials.")
|
| 168 |
+
return key
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
# --- APIエンドポイント ---
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
@app.get("/", include_in_schema=False)
|
| 175 |
+
def root():
|
| 176 |
+
"""ルートURLへのアクセスはドキュメントへリダイレクト"""
|
| 177 |
+
return RedirectResponse("/docs")
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
@app.get("/api/health")
|
| 181 |
+
def health_check():
|
| 182 |
+
"""ヘルスチェック用エンドポイント"""
|
| 183 |
+
return {"status": "ok"}
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
@app.get(
|
| 187 |
+
"/api/projects",
|
| 188 |
+
response_model=List[ProjectSummary],
|
| 189 |
+
dependencies=[Depends(get_api_key)],
|
| 190 |
+
)
|
| 191 |
+
def get_summary_data():
|
| 192 |
+
"""全企画のサマリー情報をGZIP圧縮して返す"""
|
| 193 |
+
summary_fields = {f.name for f in fields(ProjectSummary)}
|
| 194 |
+
summaries = [
|
| 195 |
+
{k: v for k, v in asdict(p).items() if k in summary_fields}
|
| 196 |
+
for p in search_pipeline.get_projects()
|
| 197 |
+
]
|
| 198 |
+
|
| 199 |
+
content = json.dumps(summaries, ensure_ascii=False).encode("utf-8")
|
| 200 |
+
return Response(
|
| 201 |
+
content=gzip.compress(content),
|
| 202 |
+
headers={"Content-Encoding": "gzip", "Content-Type": "application/json"},
|
| 203 |
+
)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
@app.get(
|
| 207 |
+
"/api/details",
|
| 208 |
+
response_model=ProjectDetail,
|
| 209 |
+
dependencies=[Depends(get_api_key)],
|
| 210 |
+
)
|
| 211 |
+
def get_project_detail(projectId: str = Query(..., description="取得したい企画のID")):
|
| 212 |
+
"""指定されたIDの企画詳細情報を返す"""
|
| 213 |
+
project = search_pipeline.get_project_map().get(projectId)
|
| 214 |
+
if not project:
|
| 215 |
+
raise HTTPException(status_code=404, detail="Project not found")
|
| 216 |
+
|
| 217 |
+
detail_fields = {f.name for f in fields(ProjectDetail)}
|
| 218 |
+
project_dict = asdict(project)
|
| 219 |
+
return {key: project_dict[key] for key in detail_fields if key in project_dict}
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
# --- 検索エンドポイント ---
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
class SearchRequest(BaseModel):
|
| 226 |
+
query: str
|
| 227 |
+
# fusion は非推奨: 設定ファイルで制御し、ここでは受け取っても無視する
|
| 228 |
+
fusion: Optional[Literal["add", "mul"]] = None
|
| 229 |
+
debug: bool = False
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
@app.post(
|
| 233 |
+
"/api/search",
|
| 234 |
+
response_model=ProjectIds,
|
| 235 |
+
dependencies=[Depends(get_api_key)],
|
| 236 |
+
)
|
| 237 |
+
def search(request: SearchRequest) -> ProjectIds:
|
| 238 |
+
"""BM25F中心の新パイプラインで検索し、企画IDを返す。"""
|
| 239 |
+
if not request.query:
|
| 240 |
+
raise HTTPException(status_code=400, detail="Query cannot be empty")
|
| 241 |
+
result_ids = search_pipeline.search(request.query, request.debug)
|
| 242 |
+
log.info(f"検索完了: {len(result_ids)} 件を返却")
|
| 243 |
+
return ProjectIds(projectIds=result_ids)
|
| 244 |
+
|
| 245 |
+
```
|