import re import unicodedata class TextNormalizer: url_pattern = re.compile(r"https?://\S+|www\.\S+", re.IGNORECASE) mention_pattern = re.compile(r"@\w+") whitespace_pattern = re.compile(r"\s+") repeated_latin_pattern = re.compile(r"([A-Za-z])\1{2,}") zero_width_pattern = re.compile(r"[\u200b-\u200f\u2060\ufeff]") repeated_punctuation_pattern = re.compile(r"([!?.,])\1{2,}") def normalize(self, text: str) -> str: normalized = unicodedata.normalize("NFKC", text).strip() normalized = self.zero_width_pattern.sub("", normalized) normalized = self.url_pattern.sub("", normalized) normalized = self.mention_pattern.sub("", normalized) normalized = self.repeated_latin_pattern.sub(r"\1\1", normalized) normalized = self.repeated_punctuation_pattern.sub(r"\1\1", normalized) normalized = self.whitespace_pattern.sub(" ", normalized) return normalized