# The following code is adapted from: https://github.com/coqui-ai/TTS/blob/dev/TTS/tts/utils/text/cleaners.py import re import regex from voxtream.utils.text.number_norm import normalize_numbers from voxtream.utils.text.time_norm import expand_time_english # Regular expression matching whitespace: _whitespace_re = re.compile(r"\s+") # List of (regular expression, replacement) pairs for abbreviations in english: pairs = [ ("mrs", "misess"), ("mr", "mister"), ("dr", "doctor"), ("st", "saint"), # ("co", "company"), ("jr", "junior"), ("maj", "major"), ("gen", "general"), ("drs", "doctors"), ("rev", "reverend"), ("lt", "lieutenant"), ("hon", "honorable"), ("sgt", "sergeant"), ("capt", "captain"), ("esq", "esquire"), ("ltd", "limited"), ("col", "colonel"), ("ft", "fort"), ("i.e.", "that is"), ("e.g.", "for example"), ("hrs", "hours"), ("mins", "minutes"), ("ms", "milliseconds"), ] abbreviations_en = [ ( re.compile(r"\b" + re.escape(key.rstrip(".")) + r"\.?(?=\W|$)", re.IGNORECASE), replacement, ) for key, replacement in pairs ] # Symbols to REPLACE (with specific substitutes) punct_replacements = { # comma-like "...": ", ", "…": ", ", ":": ",", " - ": ", ", ";": ", ", " ,": ",", "、": ",", ",": ",", # full stop-like "..": ". ", " .": ".", "?": "?", "!": "!", "。": ".", # dashes to space "—": " ", "-": " ", "ー": " ", "一": " ", "–": " ", "‘": "'", "’": "'", } # Symbols to REMOVE entirely punct_removal = ["“", "”", "«", "»", '"', "#", "*", "_", "{", "}", "¡", "¿", "|", "~"] # TODO: keep dashes -> hyphen, replace underscores with space, keep quotation marks # Build a single regex for replacements replace_pattern = re.compile("|".join(map(re.escape, punct_replacements.keys()))) remove_pattern = re.compile("[" + re.escape("".join(punct_removal)) + "]") def clean_punct(text: str) -> str: # Replace mapped symbols text = replace_pattern.sub(lambda m: punct_replacements[m.group(0)], text) # Remove unwanted symbols text = remove_pattern.sub("", text) # Remove a comma if it follows .,!? or another comma text = re.sub(r"([.,!?]),+", r"\1", text) return text def expand_abbreviations(text: str, lang: str = "en") -> str: if lang == "en": _abbreviations = abbreviations_en else: raise NotImplementedError( f"Abbreviation expansion not implemented for language: {lang}" ) for pattern, replacement in _abbreviations: text = re.sub(pattern, replacement, text) return text def lowercase(text: str) -> str: return text.lower() def remove_emojis(text: str) -> str: # Remove Emoji presentation chars text = regex.sub(r"\p{Emoji_Presentation}", "", text) # Remove extra variation selectors & modifiers (skin tone, gender, etc.) text = regex.sub(r"[\uFE0F\u200D]", "", text) return text def collapse_whitespace(text: str) -> str: return re.sub(_whitespace_re, " ", text).strip() def remove_aux_symbols(text: str) -> str: text = re.sub(r"[\<\>\(\)\[\]\"]+", "", text) return text def _normalize_internal_caps_word(match: re.Match[str]) -> str: """Lowercase internal capitals in a word while keeping all-uppercase words unchanged.""" word = match.group(0) if any(ch.isalpha() for ch in word) and word.isupper(): return word if len(word) <= 1: return word first = word[0] rest = "".join(ch.lower() if ch.isalpha() else ch for ch in word[1:]) return f"{first}{rest}" def lower_internal_capitals(text: str) -> str: """Normalize words with mixed case (e.g., VoXtream2 -> Voxtream2).""" return re.sub(r"\b[A-Za-z][A-Za-z0-9]*\b", _normalize_internal_caps_word, text) def punct_norm(text: str) -> str: """ Quick cleanup func for punctuation from LLMs or containing chars not seen often in the dataset """ text = clean_punct(text) # Expand single letter abbreviations like "A. B. C." to "A B C" text = re.sub(r"(?:(?<=\s)|^)([A-Z])\.\s*", r"\1 ", text) # remove apostrophes that are not part of a contraction text = re.sub(r"(? sci math, np.array -> np array) pattern = r"([A-Za-z]{2,})\.([A-Za-z]{2,})" text = re.sub(pattern, dot_replacer, text) return text def replace_symbols(text: str, lang: str = "en") -> str: """Replace symbols based on the lenguage tag. Args: text: Input text. lang: Lenguage identifier. ex: "en", "fr", "pt", "ca". Returns: The modified text example: input args: text: "si l'avi cau, diguem-ho" lang: "ca" Output: text: "si lavi cau, diguemho" """ if lang == "en": text = text.replace("&", " and ") text = text.replace("%", " percent ") text = text.replace("+", " plus ") text = text.replace("=", " equals ") text = text.replace("$", " dollar ") text = text.replace("/", " slash ") text = text.replace("@", " at ") elif lang == "fr": text = text.replace("&", " et ") elif lang == "pt": text = text.replace("&", " e ") elif lang == "ca": text = text.replace("&", " i ") text = text.replace("'", "") text = text.replace("-", " ") return text def english_normalizer(text: str) -> str: """Pipeline for English text, including number and abbreviation expansion.""" text = remove_emojis(text) text = punct_norm(text) # text = lowercase(text) text = expand_time_english(text) text = normalize_numbers(text) text = expand_abbreviations(text) text = lower_internal_capitals(text) text = replace_symbols(text) text = remove_aux_symbols(text) text = collapse_whitespace(text) return text