Release v1.1: PMI Phrase Merging & Smart Morphology

tokenization_df_arc.py

@@ -68,9 +68,24 @@ class MorphologicalPreTokenizer:
     PREFIXES = ['و', 'ف', 'ب', 'ك', 'ل', 'ال', 'س', 'وال', 'بال', 'كال', 'لل', 'فال']
     SUFFIXES = ['ني', 'نا', 'ك', 'كم', 'ه', 'ها', 'هم', 'هن', 'ي', 'ون', 'ين', 'ان', 'ت', 'وا', 'ة']
     
-    def __init__(self, min_stem_length: int = 2, exceptions: List[str] = None):
+    # Common entities/words to protect from segmentation (embedded fallback)
+    DEFAULT_EXCEPTIONS = {
+        "الله", "محمد", "عبدالله", "عبدالرحمن", "مكة", "بغداد", "دمشق", "القاهرة", "بيروت", "عمان",
+        "الرياض", "جدة", "الكويت", "دبي", "أبوظبي", "المنامة", "الدوحة", "مسقط", "ليبيا", "تونس",
+        "الجزائر", "المغرب", "فلسطين", "الأردن", "لبنان", "سوريا", "العراق", "مصر", "السودان", "اليمن",
+        "أمريكا", "أوروبا", "آسيا", "أفريقيا", "ترامب", "بايدن", "جوجل", "فيسبوك", "أمازون", "مايكروسوفت",
+        "أبل", "سامسونج", "سوني", "هواوي", "مرسيدس", "بي إم دبليو", "تويوتا", "هوندا", "فورد", "شيفروليه",
+        "تسلا", "ناسا", "إيلون ماسك", "مارك زوكربيرج", "بيل جيتس", "ستيف جوبز", "ألبرت أينشتاين",
+        "إسحاق نيوتن", "داروين", "بيتهوفن", "موتزارت", "شكسبير", "دوستويفسكي", "تولستوي", "نجيب محفوظ",
+        "طه حسين", "العقاد", "المنفلوطي", "جبران خليل جبران", "محمود درويش", "نزار قباني"
+    }
+
+    def __init__(self, min_stem_length: int = 2, exceptions: Optional[List[str]] = None):
         self.min_stem_length = min_stem_length
-        self.exceptions = set(exceptions) if exceptions else set()
+        # Merge user exceptions with defaults using frozenset for immutability and O(1) lookups
+        user_exceptions = set(exceptions) if exceptions else set()
+        self.exceptions = frozenset(self.DEFAULT_EXCEPTIONS.union(user_exceptions))
+        
         self.prefixes = sorted(self.PREFIXES, key=len, reverse=True)
         self.suffixes = sorted(self.SUFFIXES, key=len, reverse=True)
         self.arabic_pattern = re.compile(r'[\u0600-\u06FF]+')
@@ -108,10 +123,9 @@ class MorphologicalPreTokenizer:
     
     def segment_text(self, text: str) -> str:
         words = text.split()
-        segmented_words = []
-        for word in words:
-            segments = self.segment_word(word)
-            segmented_words.append('_'.join(segments))
+        segmented_words = [
+            '_'.join(self.segment_word(word)) for word in words
+        ]
         return ' '.join(segmented_words)
 
 class PhraseMerger:
@@ -177,18 +191,20 @@ class DFArcTokenizer(PreTrainedTokenizerFast):
     ):
         self.normalizer_helper = ArabicNormalizer(**(normalization_config or {}))
         
-        # Load exceptions if provided
-        exceptions = []
-        if exceptions_file:
+        # Load user-provided exceptions if file exists
+        user_exceptions = []
+        if exceptions_file and os.path.exists(exceptions_file):
             try:
                 with open(exceptions_file, 'r', encoding='utf-8') as f:
-                    exceptions = [line.strip() for line in f if line.strip()]
-            except FileNotFoundError:
+                    user_exceptions = [line.strip() for line in f if line.strip()]
+            except OSError:
+                # If file read fails, we just won't have custom exceptions
+                # The MorphologicalPreTokenizer has embedded defaults now.
                 pass
                 
         self.morph_helper = MorphologicalPreTokenizer(
             min_stem_length=min_stem_length,
-            exceptions=exceptions
+            exceptions=user_exceptions
         )
         self.phrase_helper = PhraseMerger(phrases_file=phrases_file)
         

tokenizer_config.json

@@ -7,7 +7,6 @@
     ]
   },
   "phrases_file": "phrase_vocab.json",
-  "exceptions_file": "exceptions.txt",
   "vocab_file": "tokenizer.json",
   "min_stem_length": 2,
   "normalization_config": {