Update utils.py

utils.py

@@ -1,10 +1,26 @@
 import re
 
+def remove_tashkeel(text):
+    tashkeel_pattern = re.compile(
+        r'[\u0617-\u061A\u064B-\u0652]'
+    )
+    return re.sub(tashkeel_pattern, '', text)
+
+def normalize_arabic(text):
+    text = re.sub("[إأآا]", "ا", text)
+    text = re.sub("ى", "ي", text)
+    text = re.sub("ؤ", "و", text)
+    text = re.sub("ئ", "ي", text)
+    text = re.sub("ة", "ه", text)
+    return text
+
 def preprocess_arabic(text):
-    text = re.sub(r"[ًٌٍَُِّْـ]", "", text)
-    text = re.sub(r"[^\w\s]", " ", text)
-    text = re.sub(r"\s+", " ", text).strip()
+    text = remove_tashkeel(text)
+    text = normalize_arabic(text)
     return text
 
 def bm25_tokenize(text):
     return preprocess_arabic(text).split()
+def preprocess_query(query):
+    query = preprocess_arabic(query)
+    return bm25_tokenize(query)