Spaces:
Runtime error
Runtime error
File size: 2,022 Bytes
d840583 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """Generic text cleaning helpers (stopwords, punctuation, URLs, numbers, ...)."""
import re
import numpy as np
from config.constants import MIN_WORDS_PER_SENTENCE, TEXT_COLUMN
from config.stopwords import get_arabic_stopwords
def remove_stop_words(text):
"""Drop NLTK Arabic stopwords from ``text``."""
arabic_stopwords = get_arabic_stopwords()
Text = [i for i in str(text).split() if i not in arabic_stopwords]
return " ".join(Text)
def Removing_non_arabic(text):
"""Replace latin letter runs with a single space."""
text = re.sub('[A-Za-z]+', ' ', text)
return text
def Removing_numbers(text):
"""Drop every digit character."""
text = ''.join([i for i in text if not i.isdigit()])
return text
def Removing_punctuations(text):
"""Replace punctuation (latin + Arabic) with spaces and squeeze whitespace."""
## Remove punctuations
text = re.sub('[%s]' % re.escape(r"""!"#$%&'()*+,،-./:;<=>؟?@[\]^_`{|}~"""), ' ', text)
text = text.replace('؛', "", )
## remove extra whitespace
text = re.sub(r'\s+', ' ', text)
text = " ".join(text.split())
return text.strip()
def Removing_urls(text):
"""Strip http(s):// and www. URLs."""
url_pattern = re.compile(r'https?://\S+|www\.\S+')
return url_pattern.sub(r'', text)
def remove_extra_Space(text):
"""Collapse repeated whitespace into single spaces."""
text = re.sub(r'\s+', ' ', text)
return " ".join(text.split())
def remove_hashtags_and_mentions(text):
"""Strip ``@mentions`` and ``#hashtags`` written with latin characters."""
text = re.sub("@[A-Za-z0-9_]+", "", text)
text = re.sub("#[A-Za-z0-9_]+", "", text)
return text
def remove_small_sentences(df, text_column=TEXT_COLUMN):
"""Turn texts shorter than 3 words into NaN, in place."""
text_position = df.columns.get_loc(text_column)
for i in range(len(df)):
if len(df[text_column].iloc[i].split()) < MIN_WORDS_PER_SENTENCE:
df.iloc[i, text_position] = np.nan
|