Fix collision benchmark: non-overlap, destructive tokenization, colon guard, hamza punctuation

src/nlp/correction_patch.py

@@ -178,12 +178,10 @@ class PatchSet:
                     overlap_start = max(patch.start_original, cs)
                     overlap_end = min(patch.end_original, ce)
                     overlap_width = overlap_end - overlap_start
-                    # Compare to the smaller patch's width
-                    patch_width = max(1, patch.end_original - patch.start_original)
-                    claimed_width = max(1, ce - cs)
-                    smaller_width = min(patch_width, claimed_width)
-                    overlap_ratio = overlap_width / smaller_width
-                    if overlap_ratio > 0.5:
+                    
+                    if overlap_width > 0:
+                        # STRICT NON-OVERLAP RULE: ANY overlap causes the lower priority patch to be dropped.
+                        # Overlapping patches cannot be safely applied sequentially by standard frontend/benchmark clients.
                         has_substantial_overlap = True
                         overlapping_resolved_idx = res_idx
                         break

src/nlp/punctuation/punctuation_rules.py

@@ -59,7 +59,22 @@ def arabic_postprocessing(text: str) -> str:
     # Fix misplaced colons for saying verbs (e.g. قال: المعلم -> قال المعلم:)
     text = re.sub(r'\b(قال|يقول|قالت|تقول|أجاب|أجابت|سأل|سألت|أخبر|أخبرت|صرح|صرحت|أضاف|أضافت|أردف|أردفت):?\s+(ال[أ-ي]+|أحمد|محمد|محمود|علي|عمر|خالد|فاطمة|مريم|عائشة|خديجة)\b:?', r'\1 \2:', text)
 
-    # Remove colons after verbs that do not introduce speech/lists
+    # NEW: Strict Colon Guard
+    _ALLOWED_COLON_CUES = r'(قال|يقول|قالت|تقول|أجاب|أجابت|سأل|سألت|أخبر|أخبرت|صرح|صرحت|أضاف|أضافت|أردف|أردفت|وضح|وضحت|أوضح|أوضحت|رد|ردت|التالي|الآتي|مثال|ملاحظة|تنبيه|تحذير|قائلا|قائلة|اسم|العمر|تاريخ|رقم|عاجل|الآتية|التالية)'
+    
+    def _colon_guard(match):
+        prev_word = match.group(1)
+        if re.fullmatch(_ALLOWED_COLON_CUES, prev_word):
+            return match.group(0)
+        # If it's a definite noun (starts with ال) and not in allowed list, it's hallucinated.
+        # e.g., "الشمس:" -> "الشمس،"
+        if prev_word.startswith('ال'):
+            return f'{prev_word}،'
+        return match.group(0)
+        
+    text = re.sub(r'([\u0600-\u06FF]+)(\s*:)', _colon_guard, text)
+    
+    # Remove colons after specific non-speech verbs (fallback for verbs without ال)
     text = re.sub(r'\b(يقدر|يستطيع|يمكن|يجب|ينبغي|يعتبر|يعد|يرى|يعتقد)\s*:', r'\1 ', text)
     # Replace semicolon with comma if followed by "و" (and) or similar conjunctions, as semicolon is for separate clauses
     text = re.sub(r'؛\s*(و|ف|ثم|أو|أم|بل)\b', r'، \1', text)

src/nlp/spelling/araspell_rules.py

@@ -298,23 +298,33 @@ class AraSpellPostProcessor:
         """
         Fix common hamza placement errors using a whitelist.
         Also handles prefixed words: و/ف/ب/ك/ل + whitelist word.
-        e.g. واصدقائي → وأصدقائي, بالاسعار → بالأسعار
+        Handles adjacent punctuation (e.g. واصدقائي، → وأصدقائي،)
         """
         words = text.split()
         result = []
         for word in words:
+            # Separate leading/trailing punctuation from the core word
+            match = re.match(r'^([\.,،؛؟!:;?\(\)\[\]«»"\'\s]*)(.*?)([\.,،؛؟!:;?\(\)\[\]«»"\'\s]*)$', word)
+            if not match or not match.group(2):
+                result.append(word)
+                continue
+                
+            lead_punct = match.group(1)
+            core_word = match.group(2)
+            trail_punct = match.group(3)
+
             # Check exact match first
-            if word in AraSpellPostProcessor.HAMZA_WHITELIST:
-                result.append(AraSpellPostProcessor.HAMZA_WHITELIST[word])
+            if core_word in AraSpellPostProcessor.HAMZA_WHITELIST:
+                result.append(lead_punct + AraSpellPostProcessor.HAMZA_WHITELIST[core_word] + trail_punct)
                 continue
 
             # Try stripping common prefixes and looking up the remainder
             fixed = False
             for prefix in AraSpellPostProcessor.HAMZA_PREFIXES:
-                if word.startswith(prefix) and len(word) > len(prefix) + 1:
-                    remainder = word[len(prefix):]
+                if core_word.startswith(prefix) and len(core_word) > len(prefix) + 1:
+                    remainder = core_word[len(prefix):]
                     if remainder in AraSpellPostProcessor.HAMZA_WHITELIST:
-                        result.append(prefix + AraSpellPostProcessor.HAMZA_WHITELIST[remainder])
+                        result.append(lead_punct + prefix + AraSpellPostProcessor.HAMZA_WHITELIST[remainder] + trail_punct)
                         fixed = True
                         break
             if not fixed:
@@ -1607,4 +1617,16 @@ class ArabicSpellChecker:
         result = AraSpellPostProcessor.fix_common_hamza(result)
         result = AraSpellPostProcessor.fix_ha_ta_marbuta(result, vocab_manager=self.vocab_manager)
 
+        # 11. DESTRUCTIVE TOKENIZATION GUARD
+        # Arabic orthography does not use standalone 1-letter words.
+        # If the model creates a standalone 1-letter word that was not in the original, it's a tokenization hallucination.
+        orig_standalone = set(w for w in original.split() if len(w) == 1)
+        res_words_list = result.split()
+        for w in res_words_list:
+            if len(w) == 1 and w not in orig_standalone:
+                if w in 'واتيبلفك':
+                    logger.info(f"[SPELLING] Blocked destructive tokenization (hallucinated standalone '{w}'): '{original}' -> '{result}'")
+                    result = original
+                    break
+
         return result