preprocessing/pipeline.py

"""The single preprocessing pipeline shared by training and inference.

Order of the steps:
1. ``replace_emoticon_with_emojis``   (emoji step)
2. ``replace_emojis_with_text``       (emoji step)
3. ``remove_stop_words``
4. ``Removing_non_arabic``
5. ``normalizeArabic``
6. ``Removing_numbers``
7. ``remove_hashtags_and_mentions``
8. ``Removing_urls``
9. ``Removing_punctuations``
10. ``Arabic_Light_Stemmer``
"""

from config.constants import TEXT_COLUMN
from preprocessing.arabic_normalizer import Arabic_Light_Stemmer, normalizeArabic
from preprocessing.emoji_handler import (
    replace_emojis_with_text,
    replace_emoticon_with_emojis,
)
from preprocessing.text_cleaning import (
    Removing_non_arabic,
    Removing_numbers,
    Removing_punctuations,
    Removing_urls,
    remove_hashtags_and_mentions,
    remove_small_sentences,
    remove_stop_words,
)

#: Steps 1-2 - emoticon/emoji substitution.
EMOJI_STEPS = (
    replace_emoticon_with_emojis,
    replace_emojis_with_text,
)

#: Steps 3-10 - applied to both the training corpus and inference input.
CLEANING_STEPS = (
    remove_stop_words,
    Removing_non_arabic,
    normalizeArabic,
    Removing_numbers,
    remove_hashtags_and_mentions,
    Removing_urls,
    Removing_punctuations,
    Arabic_Light_Stemmer,
)


def clean_text(text, convert_emojis=True):
    """Run the full cleaning chain on a single string."""
    steps = (EMOJI_STEPS + CLEANING_STEPS) if convert_emojis else CLEANING_STEPS
    for step in steps:
        text = step(text)
    return text


def preprocess_dataframe(df, text_column=TEXT_COLUMN, convert_emojis=True,
                         drop_small_sentences=True):
    df[text_column] = df[text_column].apply(
        lambda text: clean_text(text, convert_emojis=convert_emojis)
    )
    if drop_small_sentences:
        # this function will convert the text which contains one or two words into null value
        remove_small_sentences(df, text_column=text_column)
    return df