| | """Set of default text cleaners""" |
| | |
| |
|
| | import re |
| |
|
| | |
| | _whitespace_re = re.compile(r"\s+") |
| |
|
| | rep_map = { |
| | "：": ",", |
| | "；": ",", |
| | "，": ",", |
| | "。": ".", |
| | "！": "!", |
| | "？": "?", |
| | "\n": ".", |
| | "·": ",", |
| | "、": ",", |
| | "...": ".", |
| | "…": ".", |
| | "$": ".", |
| | "“": "'", |
| | "”": "'", |
| | "‘": "'", |
| | "’": "'", |
| | "（": "'", |
| | "）": "'", |
| | "(": "'", |
| | ")": "'", |
| | "《": "'", |
| | "》": "'", |
| | "【": "'", |
| | "】": "'", |
| | "[": "'", |
| | "]": "'", |
| | "—": "", |
| | "～": "-", |
| | "~": "-", |
| | "「": "'", |
| | "」": "'", |
| | } |
| |
|
| | def replace_punctuation(text): |
| | pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys())) |
| | replaced_text = pattern.sub(lambda x: rep_map[x.group()], text) |
| | return replaced_text |
| |
|
| | def lowercase(text): |
| | return text.lower() |
| |
|
| |
|
| | def collapse_whitespace(text): |
| | return re.sub(_whitespace_re, " ", text).strip() |
| |
|
| | def remove_punctuation_at_begin(text): |
| | return re.sub(r'^[,.!?]+', '', text) |
| |
|
| | def remove_aux_symbols(text): |
| | text = re.sub(r"[\<\>\(\)\[\]\"\«\»\']+", "", text) |
| | return text |
| |
|
| |
|
| | def replace_symbols(text, lang="en"): |
| | """Replace symbols based on the lenguage tag. |
| | |
| | Args: |
| | text: |
| | Input text. |
| | lang: |
| | Lenguage identifier. ex: "en", "fr", "pt", "ca". |
| | |
| | Returns: |
| | The modified text |
| | example: |
| | input args: |
| | text: "si l'avi cau, diguem-ho" |
| | lang: "ca" |
| | Output: |
| | text: "si lavi cau, diguemho" |
| | """ |
| | text = text.replace(";", ",") |
| | text = text.replace("-", " ") if lang != "ca" else text.replace("-", "") |
| | text = text.replace(":", ",") |
| | if lang == "en": |
| | text = text.replace("&", " and ") |
| | elif lang == "fr": |
| | text = text.replace("&", " et ") |
| | elif lang == "pt": |
| | text = text.replace("&", " e ") |
| | elif lang == "ca": |
| | text = text.replace("&", " i ") |
| | text = text.replace("'", "") |
| | elif lang== "es": |
| | text=text.replace("&","y") |
| | text = text.replace("'", "") |
| | return text |
| |
|
| | def unicleaners(text, cased=False, lang='en'): |
| | """Basic pipeline for Portuguese text. There is no need to expand abbreviation and |
| | numbers, phonemizer already does that""" |
| | if not cased: |
| | text = lowercase(text) |
| | text = replace_punctuation(text) |
| | text = replace_symbols(text, lang=lang) |
| | text = remove_aux_symbols(text) |
| | text = remove_punctuation_at_begin(text) |
| | text = collapse_whitespace(text) |
| | text = re.sub(r'([^\.,!\?\-…])$', r'\1.', text) |
| | return text |
| |
|
| |
|
