benchmark/normalization.py

import re
import unicodedata

def normalize_text(text: str) -> str:
    """
    Applies standard normalization:
    - Unicode compatibility (NFKC)
    - Lowercase
    - Collapses whitespaces
    - Removes punctuation while preserving digits, signs (+, -), and codes
    """
    if not text:
        return ""
    text = unicodedata.normalize("NFKC", text).lower()
    # Collapse whitespaces
    text = re.sub(r"\s+", " ", text).strip()
    # Remove standard punctuation but keep signs and alphanumeric codes
    # Keep +, -, numbers, and letters. Replace other symbols with space, then collapse
    text = re.sub(r"[^\w\s\+\-]", "", text)
    text = re.sub(r"\s+", " ", text).strip()
    return text