normalizer.py

#######3 not ued 
import re
from pyarabic import araby

def normalize_text(text):
    
    text = text.strip()
    
    # Detect if Arabic
    if re.search(r'[\u0600-\u06FF]', text):
        #  normalization
        text = araby.strip_diacritics(text)
        text = re.sub('[إأآا]', 'ا', text)
        text = text.replace('ة', 'ه')
        text = text.replace('ى', 'ي')
        text = re.sub('ـ', '', text)
    
    # Common normalization
    text = text.lower()
    text = re.sub(r"(.)\1{2,}", r"\1\1", text)  
    text = re.sub(r'\s+', ' ', text)
    
    return text