| import re | |
| import unicodedata | |
| def normalize_text(text: str) -> str: | |
| """ | |
| Applies standard normalization: | |
| - Unicode compatibility (NFKC) | |
| - Lowercase | |
| - Collapses whitespaces | |
| - Removes punctuation while preserving digits, signs (+, -), and codes | |
| """ | |
| if not text: | |
| return "" | |
| text = unicodedata.normalize("NFKC", text).lower() | |
| # Collapse whitespaces | |
| text = re.sub(r"\s+", " ", text).strip() | |
| # Remove standard punctuation but keep signs and alphanumeric codes | |
| # Keep +, -, numbers, and letters. Replace other symbols with space, then collapse | |
| text = re.sub(r"[^\w\s\+\-]", "", text) | |
| text = re.sub(r"\s+", " ", text).strip() | |
| return text | |