Create preprocessing.py

preprocessing.py

@@ -0,0 +1,20 @@
+import re
+from arabert.preprocess import ArabertPreprocessor
+model_name = "aubmindlab/bert-base-arabertv02-twitter"
+arabic_prep = ArabertPreprocessor(model_name=model_name)
+
+def clean_obfuscation(text):
+    text = str(text)
+    text = re.sub(r'https?://\S+|www\.\S+|@\S+|#', '', text)
+    text = re.sub(r'(?<=[أ-ي])[^\sأ-ي](?=[أ-ي])', '', text)
+    text = re.sub(r'(?<=[أ-ي])\s(?=[أ-ي]\s|[أ-ي]$)', '', text)
+    text = re.sub(r'ـ+', '', text)
+    text = re.sub(r'(.)\1{2,}', r'\1\1', text)
+    text = re.sub(r'[^\w\s\.]', ' ', text)
+    text = re.sub(r'\s+', ' ', text)
+    return text.strip()
+
+def full_preprocess(text):
+    text_no_trickery = clean_obfuscation(text)
+    final_text = arabic_prep.preprocess(text_no_trickery)
+    return final_text