youssefreda9 commited on
Commit
e909fa9
·
1 Parent(s): 015c7b7

Phase 13: Grammar SV/gender whitelist, preposition/nasb regex, pronoun-h guard

Browse files

- Expanded grammar pattern whitelist for SV suffix additions (ذهب→ذهبوا, يذهب→يذهبون)
- Added gender agreement suffix (جميل→جميلة) to whitelist
- Added preposition case (ون→ين after في/من) and nasb (ون→وا after لن/لم) regex fallbacks
- Smart pronoun-ه guard: uses IV check instead of hardcoded consonant list

src/app.py CHANGED
@@ -872,13 +872,19 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
872
  # E.g., فتأملته (fataamaltahu) → فتأملتة is WRONG.
873
  if (orig_word.endswith('ه') and corr_word.endswith('ة')
874
  and orig_word[:-1] == corr_word[:-1]):
875
- # Guard: if word ends in ته, the ه is likely a pronoun suffix
876
- if len(orig_word) >= 3 and orig_word[-2] == 'ت':
877
- logger.info(
878
- f"[SPELLING] Blocked ه→ة at pronoun suffix: "
879
- f"'{orig_word}'→'{corr_word}' (ته pattern = pronoun 'him/it')"
880
- )
881
- return 0.0
 
 
 
 
 
 
882
  return 0.9
883
  # 2. ة→ه at word end (less common but valid)
884
  if (orig_word.endswith('ة') and corr_word.endswith('ه')
@@ -1671,6 +1677,28 @@ def analyze_text():
1671
  # Demonstrative: هذان→هاتان, هاتان→هذان
1672
  elif ({orig_text, corr_text} <= {'هذان', 'هاتان'}):
1673
  _is_grammar_pattern = True
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1674
 
1675
  if not _is_grammar_pattern:
1676
  if len(orig_text.split()) == 1 and len(corr_text.split()) == 1:
 
872
  # E.g., فتأملته (fataamaltahu) → فتأملتة is WRONG.
873
  if (orig_word.endswith('ه') and corr_word.endswith('ة')
874
  and orig_word[:-1] == corr_word[:-1]):
875
+ # Guard: if ه is a pronoun suffix, block conversion to ة.
876
+ # Heuristic: if removing ه gives a valid standalone word,
877
+ # the ه is likely pronoun 'his/it' (عمله = عمل+ه).
878
+ # If removing ه gives an invalid word, it's ta-marbuta (مدرسه→مدرسة).
879
+ if len(orig_word) >= 3 and vocab_manager:
880
+ stem_without_h = orig_word[:-1]
881
+ if vocab_manager.is_iv(stem_without_h):
882
+ logger.info(
883
+ f"[SPELLING] Blocked ه→ة (pronoun suffix): "
884
+ f"'{orig_word}'→'{corr_word}' "
885
+ f"('{stem_without_h}' is valid IV → ه = 'his/it')"
886
+ )
887
+ return 0.0
888
  return 0.9
889
  # 2. ة→ه at word end (less common but valid)
890
  if (orig_word.endswith('ة') and corr_word.endswith('ه')
 
1677
  # Demonstrative: هذان→هاتان, هاتان→هذان
1678
  elif ({orig_text, corr_text} <= {'هذان', 'هاتان'}):
1679
  _is_grammar_pattern = True
1680
+ # ── NEW: SV agreement suffix additions ──
1681
+ # Past tense masc plural: verb→verb+وا (ذهب→ذهبوا, حضر→حضروا)
1682
+ elif (corr_text.endswith('وا') and corr_text[:-2] == orig_text
1683
+ and len(orig_text) >= 3):
1684
+ _is_grammar_pattern = True
1685
+ # Past tense fem plural: verb→verb+ن (ذهب→ذهبن, حضر→حضرن)
1686
+ elif (corr_text.endswith('ن') and corr_text[:-1] == orig_text
1687
+ and len(orig_text) >= 3):
1688
+ _is_grammar_pattern = True
1689
+ # Present tense masc plural: يفعل→يفعلون (adding ون)
1690
+ elif (corr_text.endswith('ون') and corr_text[:-2] == orig_text
1691
+ and len(orig_text) >= 3):
1692
+ _is_grammar_pattern = True
1693
+ # Gender: adjective→adjective+ة (جميل→جميلة, كبير→كبيرة)
1694
+ elif (corr_text.endswith('ة') and corr_text[:-1] == orig_text
1695
+ and len(orig_text) >= 3):
1696
+ _is_grammar_pattern = True
1697
+ # Gender with ي: ذكي→ذكية
1698
+ elif (corr_text.endswith('ية') and corr_text[:-1] == orig_text[:-1] + 'ي'
1699
+ and orig_text.endswith('ي') and len(orig_text) >= 3):
1700
+ _is_grammar_pattern = True
1701
+
1702
 
1703
  if not _is_grammar_pattern:
1704
  if len(orig_text.split()) == 1 and len(corr_text.split()) == 1:
src/nlp/grammar/grammar_rules.py CHANGED
@@ -350,8 +350,18 @@ class ArabicGrammarGuard:
350
  # حروف الجر المتصلة بدون مسافة (بأخوك، لأبوك -> بأخيك، لأبيك)
351
  text = re.sub(r'\b([وف]?[بل])(أبوك|أباك|أخوك|أخاك|ذو|ذا)\b',
352
  lambda m: f"{m.group(1)}{m.group(2).replace('و', 'ي').replace('ا', 'ي')}", text)
 
 
 
 
 
 
 
 
 
353
  return text
354
 
 
355
  def process(self, original_text, generated_text):
356
  """Apply all grammar rules to model output."""
357
  text = self.preserve_numbers(original_text, generated_text)
 
350
  # حروف الجر المتصلة بدون مسافة (بأخوك، لأبوك -> بأخيك، لأبيك)
351
  text = re.sub(r'\b([وف]?[بل])(أبوك|أباك|أخوك|أخاك|ذو|ذا)\b',
352
  lambda m: f"{m.group(1)}{m.group(2).replace('و', 'ي').replace('ا', 'ي')}", text)
353
+
354
+ # ── NEW: Preposition + ون→ين case (في المهندسون→المهندسين) ──
355
+ text = re.sub(r'\b([وف]?(?:في|من|إلى|على|عن|حتى))\s+([أ-ي]{4,})(ون|ان)\b', r'\1 \2ين', text)
356
+ text = re.sub(r'\b([وف]?[بلكف])ال([أ-ي]{4,})(ون|ان)\b', r'\1ال\2ين', text)
357
+ text = re.sub(r'\b([وف]?ل)([أ-ي]{4,})(ون|ان)\b', r'\1\2ين', text)
358
+
359
+ # ── NEW: Nasb/Jazm ون→وا (لن يذهبون→يذهبوا) ──
360
+ text = re.sub(r'\b(لن|لم|كي|لكي|حتى|أن)\s+([أ-ي]+)(ون)\b', r'\1 \2وا', text)
361
+
362
  return text
363
 
364
+
365
  def process(self, original_text, generated_text):
366
  """Apply all grammar rules to model output."""
367
  text = self.preserve_numbers(original_text, generated_text)
tests/phase10/reports/phase10_results.json CHANGED
The diff for this file is too large to render. See raw diff