| from nltk.tokenize import wordpunct_tokenize as word_tokenize |
| from nltk.tokenize import sent_tokenize |
|
|
| import re |
| import six |
| import textwrap |
|
|
| _whitelist = r"[0-9a-z\,\.\/\<\>]+" |
| _regex = "0-9a-z\,\.\/\<\>" |
|
|
|
|
| def filter_by_lang_regex(text, ratio=0.7, regex="0-9a-z\,\.\/\<\>"): |
| candidate_text = re.sub(r"[^" + regex + "]+", " ", six.ensure_str(text), flags=re.IGNORECASE).replace(" ", "") |
| text = text.replace(" ", "") |
|
|
| return (len(candidate_text) / len(text)) > ratio |
|
|
|
|
| def filter_by_num_tokens(text, gt=64): |
| return len(word_tokenize(text)) > gt |
|
|
|
|
| def filter_by_num_sents(text, gt=2): |
| return len(sent_tokenize(text)) > gt |
|
|
|
|
| def filter_by_steps(text): |
| return re.search('(step|mix all)', text, re.IGNORECASE) is not None |
|
|
|
|
| def filter_by_length(text, gt=40): |
| return len(text) > gt |
|
|
|
|
| def filter_by_item(item_list, gt=4): |
| return len(item_list) > gt |
|
|
|
|
| def chars_to_preserve(sentence, whitelist): |
| try: |
| tokenized = re.findall(whitelist, sentence, re.IGNORECASE) |
| return " ".join(tokenized) |
| except Exception as error: |
| print( |
| textwrap.dedent( |
| f""" |
| Bad characters range {whitelist}, |
| {error} |
| """ |
| ) |
| ) |
| raise |
|
|
|
|
| def normalizer(text, whitelist=r"[0-9a-z\,\.\/\<\>]+", do_lowercase=False): |
| if do_lowercase: |
| text = text.lower() |
|
|
| text = chars_to_preserve(text, whitelist=whitelist) |
| text = " ".join([word.strip() for word in text.split() if word.strip()]) |
| text = text.strip() |
|
|
| return text |
|
|
| |
| |
| |
| |
| |
| |
| |
|
|
