import re import unicodedata def normalize_text(text: str) -> str: """ Applies standard normalization: - Unicode compatibility (NFKC) - Lowercase - Collapses whitespaces - Removes punctuation while preserving digits, signs (+, -), and codes """ if not text: return "" text = unicodedata.normalize("NFKC", text).lower() # Collapse whitespaces text = re.sub(r"\s+", " ", text).strip() # Remove standard punctuation but keep signs and alphanumeric codes # Keep +, -, numbers, and letters. Replace other symbols with space, then collapse text = re.sub(r"[^\w\s\+\-]", "", text) text = re.sub(r"\s+", " ", text).strip() return text