Spaces:
Sleeping
Sleeping
ktsn-ud commited on
Commit ·
d72bce3
1
Parent(s): 881407b
不要なインポートを削除
Browse files- api/main.py +10 -8
- scripts/6_build_word_embeddings.py +7 -5
api/main.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
| 1 |
import os
|
| 2 |
import json
|
| 3 |
import gzip
|
| 4 |
-
from typing import List, Optional, Literal
|
| 5 |
|
| 6 |
from fastapi import FastAPI, Response, Depends, HTTPException, Security, Query
|
| 7 |
from fastapi.responses import RedirectResponse, JSONResponse
|
|
@@ -11,7 +10,6 @@ from pydantic import BaseModel
|
|
| 11 |
from dotenv import load_dotenv
|
| 12 |
|
| 13 |
from utils.logger import setup_logger
|
| 14 |
-
from utils.json import field_getter
|
| 15 |
from api.search.engine import SearchEngine
|
| 16 |
import schemas.projects as schema_projects
|
| 17 |
|
|
@@ -57,7 +55,7 @@ def health_check():
|
|
| 57 |
|
| 58 |
@app.get(
|
| 59 |
"/api/projects",
|
| 60 |
-
response_model=
|
| 61 |
dependencies=[Depends(get_api_key)],
|
| 62 |
)
|
| 63 |
def get_summary_data():
|
|
@@ -104,11 +102,15 @@ def search(request: SearchRequest):
|
|
| 104 |
if request.debug:
|
| 105 |
pairs, diag = result # type: ignore
|
| 106 |
ids = [pid for pid, _ in pairs]
|
| 107 |
-
return JSONResponse(
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 112 |
pairs = result # type: ignore
|
| 113 |
ids = [pid for pid, _ in pairs]
|
| 114 |
return schema_projects.ProjectIds(projectIds=ids)
|
|
|
|
| 1 |
import os
|
| 2 |
import json
|
| 3 |
import gzip
|
|
|
|
| 4 |
|
| 5 |
from fastapi import FastAPI, Response, Depends, HTTPException, Security, Query
|
| 6 |
from fastapi.responses import RedirectResponse, JSONResponse
|
|
|
|
| 10 |
from dotenv import load_dotenv
|
| 11 |
|
| 12 |
from utils.logger import setup_logger
|
|
|
|
| 13 |
from api.search.engine import SearchEngine
|
| 14 |
import schemas.projects as schema_projects
|
| 15 |
|
|
|
|
| 55 |
|
| 56 |
@app.get(
|
| 57 |
"/api/projects",
|
| 58 |
+
response_model=list[schema_projects.ProjectSummary],
|
| 59 |
dependencies=[Depends(get_api_key)],
|
| 60 |
)
|
| 61 |
def get_summary_data():
|
|
|
|
| 102 |
if request.debug:
|
| 103 |
pairs, diag = result # type: ignore
|
| 104 |
ids = [pid for pid, _ in pairs]
|
| 105 |
+
return JSONResponse(
|
| 106 |
+
content={
|
| 107 |
+
"projectIds": ids,
|
| 108 |
+
"scores": [
|
| 109 |
+
{"projectId": pid, "score": float(score)} for pid, score in pairs
|
| 110 |
+
],
|
| 111 |
+
"details": diag.get("details", []),
|
| 112 |
+
}
|
| 113 |
+
)
|
| 114 |
pairs = result # type: ignore
|
| 115 |
ids = [pid for pid, _ in pairs]
|
| 116 |
return schema_projects.ProjectIds(projectIds=ids)
|
scripts/6_build_word_embeddings.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
| 1 |
import os
|
| 2 |
import sys
|
| 3 |
import json
|
| 4 |
-
import math
|
| 5 |
from typing import Dict, List, Tuple
|
| 6 |
|
| 7 |
import numpy as np
|
|
@@ -9,14 +8,13 @@ import numpy as np
|
|
| 9 |
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
| 10 |
|
| 11 |
from utils.logger import setup_logger
|
| 12 |
-
from utils.json import
|
| 13 |
|
| 14 |
log = setup_logger(__name__)
|
| 15 |
|
| 16 |
|
| 17 |
def load_configs():
|
| 18 |
files = field_getter("config/files.json")
|
| 19 |
-
search = field_getter("config/search_model.json")
|
| 20 |
|
| 21 |
paths = {
|
| 22 |
"tf_token": files("bm25.tf_token"),
|
|
@@ -38,7 +36,9 @@ def read_vocab_from_tf_token(tf_token_path: str) -> Tuple[List[dict], set[str]]:
|
|
| 38 |
return docs, vocab
|
| 39 |
|
| 40 |
|
| 41 |
-
def stream_fasttext_vec(
|
|
|
|
|
|
|
| 42 |
"""fastText .vec から、必要語彙のみ抽出してベクトル行列を返す。"""
|
| 43 |
token_to_idx: Dict[str, int] = {}
|
| 44 |
vectors: List[np.ndarray] = []
|
|
@@ -95,7 +95,9 @@ def stream_fasttext_vec(vec_path: str, vocab: set[str]) -> Tuple[Dict[str, int],
|
|
| 95 |
|
| 96 |
|
| 97 |
def main():
|
| 98 |
-
log.info(
|
|
|
|
|
|
|
| 99 |
try:
|
| 100 |
paths = load_configs()
|
| 101 |
except Exception as e:
|
|
|
|
| 1 |
import os
|
| 2 |
import sys
|
| 3 |
import json
|
|
|
|
| 4 |
from typing import Dict, List, Tuple
|
| 5 |
|
| 6 |
import numpy as np
|
|
|
|
| 8 |
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
| 9 |
|
| 10 |
from utils.logger import setup_logger
|
| 11 |
+
from utils.json import field_getter, json_dumps
|
| 12 |
|
| 13 |
log = setup_logger(__name__)
|
| 14 |
|
| 15 |
|
| 16 |
def load_configs():
|
| 17 |
files = field_getter("config/files.json")
|
|
|
|
| 18 |
|
| 19 |
paths = {
|
| 20 |
"tf_token": files("bm25.tf_token"),
|
|
|
|
| 36 |
return docs, vocab
|
| 37 |
|
| 38 |
|
| 39 |
+
def stream_fasttext_vec(
|
| 40 |
+
vec_path: str, vocab: set[str]
|
| 41 |
+
) -> Tuple[Dict[str, int], np.ndarray]:
|
| 42 |
"""fastText .vec から、必要語彙のみ抽出してベクトル行列を返す。"""
|
| 43 |
token_to_idx: Dict[str, int] = {}
|
| 44 |
vectors: List[np.ndarray] = []
|
|
|
|
| 95 |
|
| 96 |
|
| 97 |
def main():
|
| 98 |
+
log.info(
|
| 99 |
+
"語彙/文書ベクトル(word_vocab.json, word_vectors.npz, doc_vectors.npy)を生成します"
|
| 100 |
+
)
|
| 101 |
try:
|
| 102 |
paths = load_configs()
|
| 103 |
except Exception as e:
|