preprocessing/text_cleaning.py

"""Generic text cleaning helpers (stopwords, punctuation, URLs, numbers, ...)."""

import re

import numpy as np

from config.constants import MIN_WORDS_PER_SENTENCE, TEXT_COLUMN
from config.stopwords import get_arabic_stopwords


def remove_stop_words(text):
    """Drop NLTK Arabic stopwords from ``text``."""
    arabic_stopwords = get_arabic_stopwords()
    Text = [i for i in str(text).split() if i not in arabic_stopwords]
    return " ".join(Text)


def Removing_non_arabic(text):
    """Replace latin letter runs with a single space."""
    text = re.sub('[A-Za-z]+', ' ', text)
    return text


def Removing_numbers(text):
    """Drop every digit character."""
    text = ''.join([i for i in text if not i.isdigit()])
    return text


def Removing_punctuations(text):
    """Replace punctuation (latin + Arabic) with spaces and squeeze whitespace."""
    ## Remove punctuations
    text = re.sub('[%s]' % re.escape(r"""!"#$%&'()*+,،-./:;<=>؟?@[\]^_`{|}~"""), ' ', text)
    text = text.replace('؛', "", )

    ## remove extra whitespace
    text = re.sub(r'\s+', ' ', text)
    text = " ".join(text.split())
    return text.strip()


def Removing_urls(text):
    """Strip http(s):// and www. URLs."""
    url_pattern = re.compile(r'https?://\S+|www\.\S+')
    return url_pattern.sub(r'', text)


def remove_extra_Space(text):
    """Collapse repeated whitespace into single spaces."""
    text = re.sub(r'\s+', ' ', text)
    return " ".join(text.split())


def remove_hashtags_and_mentions(text):
    """Strip ``@mentions`` and ``#hashtags`` written with latin characters."""
    text = re.sub("@[A-Za-z0-9_]+", "", text)
    text = re.sub("#[A-Za-z0-9_]+", "", text)
    return text


def remove_small_sentences(df, text_column=TEXT_COLUMN):
    """Turn texts shorter than 3 words into NaN, in place."""
    text_position = df.columns.get_loc(text_column)
    for i in range(len(df)):
        if len(df[text_column].iloc[i].split()) < MIN_WORDS_PER_SENTENCE:
            df.iloc[i, text_position] = np.nan