"""文本归一化:繁简、全角半角、谐音词、联系方式标准化""" import re import unicodedata import opencc _converter = opencc.OpenCC("t2s") # 谐音词/变体字映射(支持热更新) HOMOPHONE_MAP: dict[str, str] = { "薇": "微", "徽": "微", "威": "微", "芯": "信", "訫": "信", "加": "加", "家": "加", "嘉": "加", "V": "V", "ⅴ": "V", "∨": "V", } def _full_to_half(text: str) -> str: result = [] for ch in text: cp = ord(ch) if 0xFF01 <= cp <= 0xFF5E: result.append(chr(cp - 0xFEE0)) elif cp == 0x3000: result.append(" ") else: result.append(ch) return "".join(result) def _apply_homophone_map(text: str) -> str: for src, dst in HOMOPHONE_MAP.items(): text = text.replace(src, dst) return text def _normalize_contact(text: str) -> str: # 去除手机号中的空格/分隔符 text = re.sub(r"(\d)\s+(\d)", r"\1\2", text) text = re.sub(r"(\d)[_\-·•](\d)", r"\1\2", text) return text def normalize(text: str) -> str: text = _converter.convert(text) # 繁→简 text = unicodedata.normalize("NFKC", text) # 全角→半角 text = _full_to_half(text) text = _apply_homophone_map(text) text = _normalize_contact(text) return text