Spaces:
Sleeping
Sleeping
File size: 3,489 Bytes
ab5c984 568bbca ab5c984 568bbca ab5c984 568bbca ab5c984 568bbca ab5c984 568bbca ab5c984 568bbca ab5c984 568bbca ab5c984 568bbca ab5c984 568bbca ab5c984 568bbca ab5c984 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | import os
import sys
import json
from collections import defaultdict
from typing import Dict, List, Set
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from utils.logger import setup_logger
from utils.json import get_file_path_from_config, json_dumps
from schemas.circles import Circle
log = setup_logger(__name__)
def normalized_circle_name(name: str) -> str:
"""
サークル名の正規化を行う。
- 漢字・ひらがな・カタカナはそのまま
- 半角英数字は小文字に変換
- 記号類（！"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~・／？＜＞（）【】？｜：）を除く
"""
SYMBOLS = set("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~・／？＜＞（）【】？｜：")
if not name:
return ""
normalized = []
for char in name:
if "A" <= char <= "Z":
normalized.append(char.lower())
elif char in SYMBOLS:
continue
else:
normalized.append(char)
return "".join(normalized)
def get_substrings(texts: Dict[any, str]) -> Set[str]:
"""テキストから2文字以上の連続部分文字列をすべて抽出する。"""
substrings = set()
for key, text in texts.items():
if key == "circleId":
continue
text = text.replace(" ", "").replace(" ", "")
length = len(text)
for start in range(length):
for end in range(start + 2, length + 1):
substr = text[start:end]
substrings.add(substr)
return substrings
def main():
input_file = get_file_path_from_config("circles.circles_json")
try:
with open(input_file, encoding="utf-8") as f:
circles = json.load(f)
except FileNotFoundError:
log.error(f"入力ファイルが見つかりません: {input_file}")
sys.exit(1)
except json.JSONDecodeError as e:
log.error(f"JSONデコードエラー: {e}")
sys.exit(1)
# --- 団体名データを生成 ---
output_file_names = get_file_path_from_config("substring.circle_names")
circles = [Circle(**item) for item in circles]
circle_names = [
{
"circleId": c.circleId,
"circle": c.circleName,
"circleNormalized": normalized_circle_name(c.circleName),
"circleKana": c.circleNameKana or "",
}
for c in circles
]
# 出力先ディレクトリ作成（念のため）
os.makedirs(os.path.dirname(output_file_names), exist_ok=True)
# JSON に書き出し
json_dumps(circle_names, output_file_names)
log.info(f"サークル名データを出力しました: {output_file_names}")
# --- 部分文字列インデックスを生成 ---
output_file_substring = get_file_path_from_config("substring.substring_index")
substring_to_circle_ids: Dict[str, List[str]] = defaultdict(list)
for circle in circle_names:
circleId = circle["circleId"]
substrings = get_substrings(circle)
for substr in substrings:
substring_to_circle_ids[substr].append(circleId)
# 出力先ディレクトリ作成（念のため）
os.makedirs(os.path.dirname(output_file_substring), exist_ok=True)
# JSON に書き出し
json_dumps(substring_to_circle_ids, output_file_substring)
log.info(f"部分文字列インデックスを出力しました: {output_file_substring}")
if __name__ == "__main__":
main()
|