ktsn-ud commited on
Commit
fa87959
·
1 Parent(s): 8779d33

codex: DB連携・スキーマ変更に係る各所の変更

Browse files
app/main.py CHANGED
@@ -17,7 +17,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
17
  sys.path.append(os.path.dirname(__file__))
18
 
19
  from utils.logger import setup_logger
20
- from schemas.projects import ProjectSummary, ProjectDetail, ProjectIds
21
 
22
  from .repositories.projects_repository import ProjectsRepository
23
  from .search.engine import SearchEngine
@@ -51,6 +51,18 @@ projects_service = ProjectsService(
51
  )
52
 
53
 
 
 
 
 
 
 
 
 
 
 
 
 
54
  @asynccontextmanager
55
  async def lifespan(app: FastAPI):
56
  """アプリケーションの起動時と終了時に実行されるコード"""
@@ -93,13 +105,30 @@ def health_check():
93
  )
94
  def get_summary_data():
95
  # ここで必要なフィールドのみ抽出して返す
96
- summaries = [
97
- ProjectSummary(
98
- **item.model_dump(include=set(ProjectSummary.model_fields.keys()))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
99
  )
100
- for item in projects_service.list_projects()
101
- ]
102
- content = json.dumps(summaries, ensure_ascii=False).encode("utf-8")
103
  return Response(
104
  content=gzip.compress(content),
105
  headers={"Content-Encoding": "gzip", "Content-Type": "application/json"},
 
17
  sys.path.append(os.path.dirname(__file__))
18
 
19
  from utils.logger import setup_logger
20
+ from schemas.projects import Image, ProjectSummary, ProjectDetail, ProjectIds
21
 
22
  from .repositories.projects_repository import ProjectsRepository
23
  from .search.engine import SearchEngine
 
51
  )
52
 
53
 
54
+ def _select_primary_image(images: list[Image]) -> Image | None:
55
+ primary: Image | None = None
56
+ for image in images:
57
+ if image.order == 0:
58
+ primary = image
59
+ if primary is not None:
60
+ return primary
61
+ if images:
62
+ return images[0]
63
+ return None
64
+
65
+
66
  @asynccontextmanager
67
  async def lifespan(app: FastAPI):
68
  """アプリケーションの起動時と終了時に実行されるコード"""
 
105
  )
106
  def get_summary_data():
107
  # ここで必要なフィールドのみ抽出して返す
108
+ summaries_payload: list[dict[str, object]] = []
109
+ for project in projects_service.list_projects():
110
+ # 画像はprimaryの1枚だけ返す
111
+ image = _select_primary_image(project.images)
112
+ if not image:
113
+ log.error("企画ID %s の画像選択に失敗しました", project.projectId)
114
+
115
+ summary = ProjectSummary(
116
+ projectId=project.projectId,
117
+ circleName=project.circleName,
118
+ name=project.name,
119
+ category=project.category,
120
+ firstDay=project.firstDay,
121
+ secondDay=project.secondDay,
122
+ thirdDay=project.thirdDay,
123
+ location=project.location,
124
+ description=project.description,
125
+ prSummary=project.prSummary,
126
+ notes=project.notes,
127
+ image=image,
128
  )
129
+ summaries_payload.append(summary.model_dump(mode="json"))
130
+
131
+ content = json.dumps(summaries_payload, ensure_ascii=False).encode("utf-8")
132
  return Response(
133
  content=gzip.compress(content),
134
  headers={"Content-Encoding": "gzip", "Content-Type": "application/json"},
app/repositories/_data_example.py CHANGED
@@ -12,10 +12,10 @@
12
  "description": "自律走行ロボットの展示とデモを行います。",
13
  "prSummary": "ロボの魅力をぎゅっと!",
14
  "prDetail": "部員が制作した各種ロボットを分かりやすく紹介します。体験コーナーもあり。",
15
- "note": ["写真撮影OK", "混雑時は入場制限あり"],
16
  "images": [
17
- {"filename": "rb001", "extension": "webp", "order": 1},
18
- {"filename": "rb002", "extension": "webp", "order": 2},
19
  ],
20
  "urls": [
21
  {"service": "X", "url": "https://x.com/roboken"},
@@ -34,10 +34,10 @@
34
  "location": "野外ステージ",
35
  # texts.* は該当APPROVEDが無い場合 None になり得る
36
  "description": None,
37
- "prSummary": None,
38
  "prDetail": None,
39
  # COALESCE により空集合は [] になる
40
- "note": [],
41
  "images": [],
42
  "urls": [],
43
  },
 
12
  "description": "自律走行ロボットの展示とデモを行います。",
13
  "prSummary": "ロボの魅力をぎゅっと!",
14
  "prDetail": "部員が制作した各種ロボットを分かりやすく紹介します。体験コーナーもあり。",
15
+ "notes": ["写真撮影OK", "混雑時は入場制限あり"],
16
  "images": [
17
+ {"filename": "rb001", "extension": "webp", "order": 0},
18
+ {"filename": "rb002", "extension": "webp", "order": 1},
19
  ],
20
  "urls": [
21
  {"service": "X", "url": "https://x.com/roboken"},
 
34
  "location": "野外ステージ",
35
  # texts.* は該当APPROVEDが無い場合 None になり得る
36
  "description": None,
37
+ "prSummary": "",
38
  "prDetail": None,
39
  # COALESCE により空集合は [] になる
40
+ "notes": [],
41
  "images": [],
42
  "urls": [],
43
  },
app/repositories/projects_repository.py CHANGED
@@ -1,3 +1,4 @@
 
1
  from typing import Any, Dict, List
2
 
3
  from pymysql import MySQLError
@@ -47,6 +48,29 @@ class ProjectsRepository:
47
  "MUSIC": "音楽",
48
  "OTHER": "その他",
49
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  return Project(
51
  projectId=row["projectId"],
52
  circleName=row["circleName"],
@@ -60,7 +84,7 @@ class ProjectsRepository:
60
  description=row["description"],
61
  prSummary=row["prSummary"],
62
  prDetail=row["prDetail"],
63
- notes=row["notes"],
64
- images=[Image(**img) for img in row["images"]],
65
- urls=[Sns(**url) for url in row["urls"]],
66
  )
 
1
+ import json
2
  from typing import Any, Dict, List
3
 
4
  from pymysql import MySQLError
 
48
  "MUSIC": "音楽",
49
  "OTHER": "その他",
50
  }
51
+ def _coerce_list(value: Any) -> list[Any]:
52
+ if value is None:
53
+ return []
54
+ if isinstance(value, list):
55
+ return value
56
+ if isinstance(value, str):
57
+ try:
58
+ parsed = json.loads(value)
59
+ except json.JSONDecodeError:
60
+ return []
61
+ return parsed if isinstance(parsed, list) else []
62
+ return []
63
+
64
+ def _coerce_dict_list(value: Any) -> list[Dict[str, Any]]:
65
+ return [item for item in _coerce_list(value) if isinstance(item, dict)]
66
+
67
+ def _coerce_str_list(value: Any) -> list[str]:
68
+ items = []
69
+ for item in _coerce_list(value):
70
+ if isinstance(item, str):
71
+ items.append(item)
72
+ return items
73
+
74
  return Project(
75
  projectId=row["projectId"],
76
  circleName=row["circleName"],
 
84
  description=row["description"],
85
  prSummary=row["prSummary"],
86
  prDetail=row["prDetail"],
87
+ notes=_coerce_str_list(row.get("notes")) or None,
88
+ images=[Image(**img) for img in _coerce_dict_list(row.get("images"))],
89
+ urls=[Sns(**url) for url in _coerce_dict_list(row.get("urls"))],
90
  )
app/repositories/query.sql CHANGED
@@ -11,7 +11,7 @@ SELECT
11
  texts.description AS "description",
12
  texts.prSummary AS "prSummary",
13
  texts.prDetail AS "prDetail",
14
- COALESCE(visitor_note.note, '[]'::jsonb) AS "notes",
15
  COALESCE(images."images", '[]'::jsonb) AS "images",
16
  COALESCE(sns.urls, '[]'::jsonb) AS "urls"
17
 
@@ -56,7 +56,7 @@ LEFT JOIN LATERAL (
56
 
57
  -- 来場者向けタグ情報(注意事項)を結合
58
  LEFT JOIN LATERAL (
59
- SELECT array_agg(t.text ORDER BY cpt.id ASC) AS note
60
  FROM public."CPTag" AS cpt
61
  JOIN public."Tag" AS t
62
  ON t.id = cpt."tagId"
 
11
  texts.description AS "description",
12
  texts.prSummary AS "prSummary",
13
  texts.prDetail AS "prDetail",
14
+ COALESCE(visitor_note.notes, '[]'::jsonb) AS "notes",
15
  COALESCE(images."images", '[]'::jsonb) AS "images",
16
  COALESCE(sns.urls, '[]'::jsonb) AS "urls"
17
 
 
56
 
57
  -- 来場者向けタグ情報(注意事項)を結合
58
  LEFT JOIN LATERAL (
59
+ SELECT array_agg(t.text ORDER BY cpt.id ASC) AS notes
60
  FROM public."CPTag" AS cpt
61
  JOIN public."Tag" AS t
62
  ON t.id = cpt."tagId"
app/search/engine.py CHANGED
@@ -169,7 +169,7 @@ class SearchEngine:
169
  p["projectId"]: normalize_text_for_org(p.get("circleName") or "")
170
  for p in self.projects
171
  }
172
- self.circleNameKana_norms = {
173
  p["projectId"]: normalize_text_for_org(p.get("circleNameKana") or "")
174
  for p in self.projects
175
  }
@@ -641,8 +641,8 @@ class SearchEngine:
641
  details.append(
642
  {
643
  "projectId": ids[idx],
644
- "organization": project.get("organization"),
645
- "title": project.get("title"),
646
  "bm25": float(bm25[idx]),
647
  "ws_filter_topk": float(ws_filter[idx]),
648
  "ws_rerank_pairavg": float(ws_rerank[idx])
 
169
  p["projectId"]: normalize_text_for_org(p.get("circleName") or "")
170
  for p in self.projects
171
  }
172
+ self.reading_norms = {
173
  p["projectId"]: normalize_text_for_org(p.get("circleNameKana") or "")
174
  for p in self.projects
175
  }
 
641
  details.append(
642
  {
643
  "projectId": ids[idx],
644
+ "circleName": project.get("circleName"),
645
+ "name": project.get("name"),
646
  "bm25": float(bm25[idx]),
647
  "ws_filter_topk": float(ws_filter[idx]),
648
  "ws_rerank_pairavg": float(ws_rerank[idx])
config/files.json CHANGED
@@ -8,8 +8,8 @@
8
  "stopwords": "resources/stopwords.json"
9
  },
10
  "projects": {
11
- "original_csv": "resources/projects_original.csv",
12
- "projects_json": "data/generated/projects.json"
13
  },
14
  "bm25": {
15
  "bm25_meta": "data/generated/bm25_meta.json",
 
8
  "stopwords": "resources/stopwords.json"
9
  },
10
  "projects": {
11
+ "projects_json": "data/generated/projects.json",
12
+ "output_json": "data/generated/projects.json"
13
  },
14
  "bm25": {
15
  "bm25_meta": "data/generated/bm25_meta.json",
config/search_model.json CHANGED
@@ -10,8 +10,8 @@
10
  "circleName",
11
  "circleNameKana",
12
  "description",
13
- "prComment",
14
- "prCommentLong"
15
  ],
16
  "synonyms": {
17
  "enable": true,
@@ -38,8 +38,8 @@
38
  "circleName": 1.5,
39
  "circleNameKana": 0.6,
40
  "description": 1.0,
41
- "prComment": 1.0,
42
- "prCommentLong": 0.8
43
  }
44
  },
45
  "word_sim": {
 
10
  "circleName",
11
  "circleNameKana",
12
  "description",
13
+ "prSummary",
14
+ "prDetail"
15
  ],
16
  "synonyms": {
17
  "enable": true,
 
38
  "circleName": 1.5,
39
  "circleNameKana": 0.6,
40
  "description": 1.0,
41
+ "prSummary": 1.0,
42
+ "prDetail": 0.8
43
  }
44
  },
45
  "word_sim": {
docs/project_schema_change_points.md CHANGED
@@ -6,8 +6,7 @@
6
  - The API layer (`app/main.py`) serves the DB-derived models directly, while the search engine (`app/search/engine.py`) consumes the statically generated artefacts.
7
 
8
  ## Config (`config/files.json`)
9
- - `config/files.json:11-12` still references `projects.original_csv`; this key can be retired once the CSV path is unused.
10
- - Add an explicit output mapping (e.g. `projects.output_json`) so scripts resolve the generated JSON path without relying on hard-coded fallbacks. Update all callers to reference the new key.
11
 
12
  ## Batch Scripts (`scripts/`)
13
  ### `scripts/2_create_projects_data.py`
@@ -33,7 +32,7 @@
33
  - Ensure the summary payload exposes `notes` (renamed from `note`) and the PR fields `prSummary` / `prDetail` as returned by the repository.
34
 
35
  ## Search Engine (`app/search/engine.py`)
36
- - In `SearchEngine.initialize` (`app/search/engine.py:164-175`) we populate kana norms into `self.circleNameKana_norms`, but the boosting logic reads `self.reading_norms` (`app/search/engine.py:545`). Assign the kana norms into `self.reading_norms` so substring boosts still consider kana.
37
  - Debug responses at `app/search/engine.py:643-645` reference `organization` / `title`; switch these to `circleName` / `name` so diagnostics stay meaningful with the new schema.
38
  - Rebuild all search assets (`projects.json`, `tf_token.json`, `bm25_meta.json`, `substring_index.json`) after applying the script updates above.
39
 
 
6
  - The API layer (`app/main.py`) serves the DB-derived models directly, while the search engine (`app/search/engine.py`) consumes the statically generated artefacts.
7
 
8
  ## Config (`config/files.json`)
9
+ - Remove the unused `projects.original_csv` entry and expose an explicit `projects.output_json` mapping so scripts resolve the generated JSON path without relying on hard-coded fallbacks.
 
10
 
11
  ## Batch Scripts (`scripts/`)
12
  ### `scripts/2_create_projects_data.py`
 
32
  - Ensure the summary payload exposes `notes` (renamed from `note`) and the PR fields `prSummary` / `prDetail` as returned by the repository.
33
 
34
  ## Search Engine (`app/search/engine.py`)
35
+ - In `SearchEngine.initialize` (`app/search/engine.py:164-175`) ensure kana norms are stored directly in `self.reading_norms` so substring boosts continue to consider kana values during scoring.
36
  - Debug responses at `app/search/engine.py:643-645` reference `organization` / `title`; switch these to `circleName` / `name` so diagnostics stay meaningful with the new schema.
37
  - Rebuild all search assets (`projects.json`, `tf_token.json`, `bm25_meta.json`, `substring_index.json`) after applying the script updates above.
38
 
schemas/projects.py CHANGED
@@ -60,7 +60,7 @@ class Project(BaseModel):
60
  prSummary: str
61
  prDetail: Optional[str]
62
  notes: Optional[list[str]]
63
- images: list[Image]
64
  urls: list[Sns]
65
 
66
 
@@ -93,7 +93,7 @@ class ProjectSummary(BaseModel):
93
  description: str
94
  prSummary: str
95
  notes: Optional[list[str]]
96
- image: Image
97
 
98
 
99
  class ProjectDetail(BaseModel):
@@ -128,7 +128,7 @@ class ProjectDetail(BaseModel):
128
  prSummary: str
129
  prDetail: Optional[str]
130
  notes: Optional[list[str]]
131
- images: list[Image]
132
  urls: list[Sns]
133
 
134
 
 
60
  prSummary: str
61
  prDetail: Optional[str]
62
  notes: Optional[list[str]]
63
+ images: Optional[list[Image]]
64
  urls: list[Sns]
65
 
66
 
 
93
  description: str
94
  prSummary: str
95
  notes: Optional[list[str]]
96
+ image: Optional[Image]
97
 
98
 
99
  class ProjectDetail(BaseModel):
 
128
  prSummary: str
129
  prDetail: Optional[str]
130
  notes: Optional[list[str]]
131
+ images: Optional[list[Image]]
132
  urls: list[Sns]
133
 
134
 
schemas/tf_token.py CHANGED
@@ -11,8 +11,8 @@ class Fields(BaseModel):
11
  circleName: TfOfField
12
  circleNameKana: TfOfField
13
  description: TfOfField
14
- prComment: TfOfField
15
- prCommentLong: TfOfField
16
 
17
 
18
  class Project(BaseModel):
 
11
  circleName: TfOfField
12
  circleNameKana: TfOfField
13
  description: TfOfField
14
+ prSummary: TfOfField
15
+ prDetail: TfOfField
16
 
17
 
18
  class Project(BaseModel):
scripts/2_create_projects_data.py CHANGED
@@ -28,8 +28,10 @@ def main():
28
  log.error(f"企画データの取得に失敗しました: {exc}")
29
  sys.exit(1)
30
 
 
 
31
  os.makedirs(os.path.dirname(output_json_path), exist_ok=True)
32
- json_dumps(projects, output_json_path)
33
 
34
  log.info(f"企画データの生成が完了しました: {output_json_path}")
35
  sys.exit(0)
 
28
  log.error(f"企画データの取得に失敗しました: {exc}")
29
  sys.exit(1)
30
 
31
+ project_dicts = [project.model_dump(mode="json") for project in projects]
32
+
33
  os.makedirs(os.path.dirname(output_json_path), exist_ok=True)
34
+ json_dumps(project_dicts, output_json_path)
35
 
36
  log.info(f"企画データの生成が完了しました: {output_json_path}")
37
  sys.exit(0)
scripts/5_prepare_tf_token.py CHANGED
@@ -106,12 +106,12 @@ def main():
106
  return field_objs.get(name, tf_token.TfOfField(len=0, tf={}))
107
 
108
  fields_obj = tf_token.Fields(
109
- title=get_field("title"),
110
  circleName=get_field("circleName"),
111
  circleNameKana=get_field("circleNameKana"),
112
  description=get_field("description"),
113
- prComment=get_field("prComment"),
114
- prCommentLong=get_field("prCommentLong"),
115
  )
116
 
117
  project_entry = tf_token.Project(
 
106
  return field_objs.get(name, tf_token.TfOfField(len=0, tf={}))
107
 
108
  fields_obj = tf_token.Fields(
109
+ name=get_field("name"),
110
  circleName=get_field("circleName"),
111
  circleNameKana=get_field("circleNameKana"),
112
  description=get_field("description"),
113
+ prSummary=get_field("prSummary"),
114
+ prDetail=get_field("prDetail"),
115
  )
116
 
117
  project_entry = tf_token.Project(
scripts/7_prepare_circle_names.py CHANGED
@@ -68,9 +68,9 @@ def main():
68
  circle_names = [
69
  {
70
  "projectId": p.projectId,
71
- "circle": p.organization,
72
- "circleNormalized": normalized_circle_name(p.organization),
73
- "circleKana": p.reading,
74
  }
75
  for p in projects
76
  ]
 
68
  circle_names = [
69
  {
70
  "projectId": p.projectId,
71
+ "circle": p.circleName,
72
+ "circleNormalized": normalized_circle_name(p.circleName),
73
+ "circleKana": p.circleNameKana or "",
74
  }
75
  for p in projects
76
  ]
tests/repositories/test_projects_repository.py CHANGED
@@ -1,3 +1,4 @@
 
1
  import os
2
  import sys
3
  import types
@@ -74,43 +75,42 @@ class ProjectsRepositoryTestCase(unittest.TestCase):
74
  def test_transform_row_maps_database_columns_to_api_keys(self):
75
  repo = ProjectsRepository()
76
  db_row = {
77
- "project_id": "100",
78
- "circle_name": "Circle A",
79
- "circle_name_kana": "サークルエー",
80
  "name": "Project A",
81
- "category": "展示",
82
- "first_day": True,
83
- "second_day": False,
84
- "third_day": True,
85
  "location": "Room 1",
86
  "description": "Description",
87
- "pr_comment": "PR",
88
- "pr_comment_long": "Long PR",
89
- "note": None,
90
- "image_name": "img.png",
91
- "optional_image_name1": None,
92
- "optional_image_name2": None,
93
- "optional_image_name3": None,
94
- "optional_image_name4": None,
95
- "optional_image_name5": None,
96
- "url_x": "https://x.com",
97
- "url_instagram": None,
98
- "url_official_website": None,
99
- "url_youtube": None,
100
- "url_others": None,
101
  }
102
 
103
  api_row = repo._transform_row(db_row)
104
 
105
  self.assertEqual(api_row.projectId, "100")
106
  self.assertEqual(api_row.circleName, "Circle A")
107
- self.assertEqual(api_row.category, "展示")
108
  self.assertTrue(api_row.firstDay)
109
- self.assertIsNone(api_row.optionalImageName1)
 
 
110
 
111
  def test_transform_row_handles_missing_columns(self):
112
  repo = ProjectsRepository()
113
- db_row = {"project_id": "200"}
114
 
115
  # 必須フィールドが欠けているため、ValidationError が発生するはず
116
  self.assertRaises(ValidationError, lambda: repo._transform_row(db_row))
 
1
+ import json
2
  import os
3
  import sys
4
  import types
 
75
  def test_transform_row_maps_database_columns_to_api_keys(self):
76
  repo = ProjectsRepository()
77
  db_row = {
78
+ "projectId": "100",
79
+ "circleName": "Circle A",
80
+ "circleNameKana": "サークルエー",
81
  "name": "Project A",
82
+ "category": "PERFORMANCE",
83
+ "firstDay": True,
84
+ "secondDay": False,
85
+ "thirdDay": True,
86
  "location": "Room 1",
87
  "description": "Description",
88
+ "prSummary": "PR",
89
+ "prDetail": "Long PR",
90
+ "notes": json.dumps(["注意事項"]),
91
+ "images": [
92
+ {"filename": "primary", "extension": "png", "order": 0},
93
+ {"filename": "sub", "extension": "png", "order": 1},
94
+ ],
95
+ "urls": json.dumps([
96
+ {"service": "X", "url": "https://x.com"},
97
+ {"service": "Website", "url": "https://example.com"},
98
+ ]),
 
 
 
99
  }
100
 
101
  api_row = repo._transform_row(db_row)
102
 
103
  self.assertEqual(api_row.projectId, "100")
104
  self.assertEqual(api_row.circleName, "Circle A")
105
+ self.assertEqual(api_row.category, "パフォーマンス")
106
  self.assertTrue(api_row.firstDay)
107
+ self.assertEqual(api_row.notes, ["注意事項"])
108
+ self.assertEqual(api_row.images[0].filename, "primary")
109
+ self.assertEqual(api_row.urls[0].service, "X")
110
 
111
  def test_transform_row_handles_missing_columns(self):
112
  repo = ProjectsRepository()
113
+ db_row = {"projectId": "200"}
114
 
115
  # 必須フィールドが欠けているため、ValidationError が発生するはず
116
  self.assertRaises(ValidationError, lambda: repo._transform_row(db_row))
tests/services/test_projects_service.py CHANGED
@@ -82,20 +82,17 @@ class FakeProjectsRepository:
82
  "thirdDay": True,
83
  "location": "Room 1",
84
  "description": "Description",
85
- "prComment": "PR",
86
- "prCommentLong": "Long PR",
87
- "note": None,
88
- "imageName": "img.png",
89
- "optionalImageName1": None,
90
- "optionalImageName2": None,
91
- "optionalImageName3": None,
92
- "optionalImageName4": None,
93
- "optionalImageName5": None,
94
- "urlX": "https://x.com",
95
- "urlInstagram": None,
96
- "urlOfficialWebsite": None,
97
- "urlYoutube": None,
98
- "urlOthers": None,
99
  }
100
  projects = [Project(**p)]
101
  self._projects = projects
 
82
  "thirdDay": True,
83
  "location": "Room 1",
84
  "description": "Description",
85
+ "prSummary": "PR",
86
+ "prDetail": "Long PR",
87
+ "notes": ["注意事項"],
88
+ "images": [
89
+ {"filename": "img", "extension": "png", "order": 0},
90
+ {"filename": "img2", "extension": "png", "order": 1},
91
+ ],
92
+ "urls": [
93
+ {"service": "X", "url": "https://x.com"},
94
+ {"service": "Website", "url": "https://example.com"},
95
+ ],
 
 
 
96
  }
97
  projects = [Project(**p)]
98
  self._projects = projects