preprocessing/arabic_normalizer.py

"""Arabic-specific normalization and light stemming."""

import re
import warnings

with warnings.catch_warnings():
    warnings.simplefilter("ignore", SyntaxWarning)
    import pyarabic.araby as araby
    from tashaphyne.stemming import ArabicLightStemmer


def normalizeArabic(text):
    """Unify Arabic letter forms, collapse repetitions and strip diacritics/digits."""
    text = text.strip()
    text = re.sub("ى", "ي", text)
    text = re.sub("ؤ", "ء", text)
    text = re.sub("ئ", "ء", text)
    text = re.sub("ة", "ه", text)

    #remove repetetions
    text = re.sub("[إأٱآا]", "ا", text)
    text = text.replace('وو', 'و')
    text = text.replace('يي', 'ي')
    text = text.replace('ييي', 'ي')
    text = text.replace('اا', 'ا')

    ## remove extra whitespace
    text = re.sub(r'\s+', ' ', text)

    # Remove longation
    text = re.sub(r'(.)\1+', r"\1\1", text)

    # Strip vowels from a text, include Shadda.
    text = araby.strip_tashkeel(text)

    # Strip diacritics from a text, include harakats and small lettres The striped marks are
    text = araby.strip_diacritics(text)
    text = ''.join([i for i in text if not i.isdigit()])
    return text


def Arabic_Light_Stemmer(text):
    """Light-stem every word of ``text`` with Tashaphyne's ``ArabicLightStemmer``."""
    # Arabic Light Stemming is a specific type of stemming applied to Arabic words. Stemming is the process
    # of reducing words to their root or base form, which helps in normalizing text for analysis.
    Arabic_Stemmer = ArabicLightStemmer()

    # stemming each word
    text = [Arabic_Stemmer.light_stem(y) for y in text.split()]

    return " ".join(text)