src/understanding/normalizer.py

"""文本归一化：繁简、全角半角、谐音词、联系方式标准化"""
import re
import unicodedata
import opencc

_converter = opencc.OpenCC("t2s")

# 谐音词/变体字映射（支持热更新）
HOMOPHONE_MAP: dict[str, str] = {
    "薇": "微", "徽": "微", "威": "微",
    "芯": "信", "訫": "信",
    "加": "加", "家": "加", "嘉": "加",
    "V": "V", "ⅴ": "V", "∨": "V",
}


def _full_to_half(text: str) -> str:
    result = []
    for ch in text:
        cp = ord(ch)
        if 0xFF01 <= cp <= 0xFF5E:
            result.append(chr(cp - 0xFEE0))
        elif cp == 0x3000:
            result.append(" ")
        else:
            result.append(ch)
    return "".join(result)


def _apply_homophone_map(text: str) -> str:
    for src, dst in HOMOPHONE_MAP.items():
        text = text.replace(src, dst)
    return text


def _normalize_contact(text: str) -> str:
    # 去除手机号中的空格/分隔符
    text = re.sub(r"(\d)\s+(\d)", r"\1\2", text)
    text = re.sub(r"(\d)[_\-·•](\d)", r"\1\2", text)
    return text


def normalize(text: str) -> str:
    text = _converter.convert(text)           # 繁→简
    text = unicodedata.normalize("NFKC", text)  # 全角→半角
    text = _full_to_half(text)
    text = _apply_homophone_map(text)
    text = _normalize_contact(text)
    return text