arabic_processor.py

import re
from pyarabic import araby


class ArabicProcessor:
    def __init__(self):
        
       
        pass



    # clean , normalization 
    def normalize_arabic(self, text: str) -> str:
        

        if not text:
            return ""

        #  remove diacritics (تشكيل)
        text = araby.strip_diacritics(text)

        #  unify forms (إ أ آ -> ا)
        text = re.sub(r'[إأآا]', 'ا', text)

        #  normalize 
        text = text.replace('ى', 'ي')

        # normalize 
        text = text.replace('ة', 'ه')

        #  remove  (ـ)
        text = text.replace('ـ', '')

        #  remove punctuation 
        text = re.sub(r'[^\w\s]', ' ', text)

        #  normalize whitespace
        text = re.sub(r'\s+', ' ', text)

        return text.strip().lower()

   



    # query expansion
    def expand_query_arabic(self, query: str):
        

        ##### rely on embedding

        if not query:
            return []

        query = self.normalize_arabic(query)

        expanded = set()
        expanded.add(query)

        words = query.split()

        



        ### sub query generation
        if len(words) > 1:
            for i in range(len(words)):
                sub_query = " ".join(words[:i] + words[i+1:])
                if len(sub_query.split()) >= 1:
                    expanded.add(sub_query)



        
        #Prefix / suffix variations 
        
        expanded.add("مشكلة " + query)
        expanded.add("حل " + query)
        expanded.add("عن " + query)
        expanded.add("كيفية " + query)
        expanded.add("طريقة " + query)
        expanded.add("خطوات " + query)
        expanded.add("اريد معرفة" + query)


        


        # keep expanded
        
        return list(expanded)