Commit ·
e909fa9
1
Parent(s): 015c7b7
Phase 13: Grammar SV/gender whitelist, preposition/nasb regex, pronoun-h guard
Browse files- Expanded grammar pattern whitelist for SV suffix additions (ذهب→ذهبوا, يذهب→يذهبون)
- Added gender agreement suffix (جميل→جميلة) to whitelist
- Added preposition case (ون→ين after في/من) and nasb (ون→وا after لن/لم) regex fallbacks
- Smart pronoun-ه guard: uses IV check instead of hardcoded consonant list
- src/app.py +35 -7
- src/nlp/grammar/grammar_rules.py +10 -0
- tests/phase10/reports/phase10_results.json +0 -0
src/app.py
CHANGED
|
@@ -872,13 +872,19 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
|
|
| 872 |
# E.g., فتأملته (fataamaltahu) → فتأملتة is WRONG.
|
| 873 |
if (orig_word.endswith('ه') and corr_word.endswith('ة')
|
| 874 |
and orig_word[:-1] == corr_word[:-1]):
|
| 875 |
-
# Guard: if
|
| 876 |
-
if
|
| 877 |
-
|
| 878 |
-
|
| 879 |
-
|
| 880 |
-
|
| 881 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 882 |
return 0.9
|
| 883 |
# 2. ة→ه at word end (less common but valid)
|
| 884 |
if (orig_word.endswith('ة') and corr_word.endswith('ه')
|
|
@@ -1671,6 +1677,28 @@ def analyze_text():
|
|
| 1671 |
# Demonstrative: هذان→هاتان, هاتان→هذان
|
| 1672 |
elif ({orig_text, corr_text} <= {'هذان', 'هاتان'}):
|
| 1673 |
_is_grammar_pattern = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1674 |
|
| 1675 |
if not _is_grammar_pattern:
|
| 1676 |
if len(orig_text.split()) == 1 and len(corr_text.split()) == 1:
|
|
|
|
| 872 |
# E.g., فتأملته (fataamaltahu) → فتأملتة is WRONG.
|
| 873 |
if (orig_word.endswith('ه') and corr_word.endswith('ة')
|
| 874 |
and orig_word[:-1] == corr_word[:-1]):
|
| 875 |
+
# Guard: if ه is a pronoun suffix, block conversion to ة.
|
| 876 |
+
# Heuristic: if removing ه gives a valid standalone word,
|
| 877 |
+
# the ه is likely pronoun 'his/it' (عمله = عمل+ه).
|
| 878 |
+
# If removing ه gives an invalid word, it's ta-marbuta (مدرسه→مدرسة).
|
| 879 |
+
if len(orig_word) >= 3 and vocab_manager:
|
| 880 |
+
stem_without_h = orig_word[:-1]
|
| 881 |
+
if vocab_manager.is_iv(stem_without_h):
|
| 882 |
+
logger.info(
|
| 883 |
+
f"[SPELLING] Blocked ه→ة (pronoun suffix): "
|
| 884 |
+
f"'{orig_word}'→'{corr_word}' "
|
| 885 |
+
f"('{stem_without_h}' is valid IV → ه = 'his/it')"
|
| 886 |
+
)
|
| 887 |
+
return 0.0
|
| 888 |
return 0.9
|
| 889 |
# 2. ة→ه at word end (less common but valid)
|
| 890 |
if (orig_word.endswith('ة') and corr_word.endswith('ه')
|
|
|
|
| 1677 |
# Demonstrative: هذان→هاتان, هاتان→هذان
|
| 1678 |
elif ({orig_text, corr_text} <= {'هذان', 'هاتان'}):
|
| 1679 |
_is_grammar_pattern = True
|
| 1680 |
+
# ── NEW: SV agreement suffix additions ──
|
| 1681 |
+
# Past tense masc plural: verb→verb+وا (ذهب→ذهبوا, حضر→حضروا)
|
| 1682 |
+
elif (corr_text.endswith('وا') and corr_text[:-2] == orig_text
|
| 1683 |
+
and len(orig_text) >= 3):
|
| 1684 |
+
_is_grammar_pattern = True
|
| 1685 |
+
# Past tense fem plural: verb→verb+ن (ذهب→ذهبن, حضر→حضرن)
|
| 1686 |
+
elif (corr_text.endswith('ن') and corr_text[:-1] == orig_text
|
| 1687 |
+
and len(orig_text) >= 3):
|
| 1688 |
+
_is_grammar_pattern = True
|
| 1689 |
+
# Present tense masc plural: يفعل→يفعلون (adding ون)
|
| 1690 |
+
elif (corr_text.endswith('ون') and corr_text[:-2] == orig_text
|
| 1691 |
+
and len(orig_text) >= 3):
|
| 1692 |
+
_is_grammar_pattern = True
|
| 1693 |
+
# Gender: adjective→adjective+ة (جميل→جميلة, كبير→كبيرة)
|
| 1694 |
+
elif (corr_text.endswith('ة') and corr_text[:-1] == orig_text
|
| 1695 |
+
and len(orig_text) >= 3):
|
| 1696 |
+
_is_grammar_pattern = True
|
| 1697 |
+
# Gender with ي: ذكي→ذكية
|
| 1698 |
+
elif (corr_text.endswith('ية') and corr_text[:-1] == orig_text[:-1] + 'ي'
|
| 1699 |
+
and orig_text.endswith('ي') and len(orig_text) >= 3):
|
| 1700 |
+
_is_grammar_pattern = True
|
| 1701 |
+
|
| 1702 |
|
| 1703 |
if not _is_grammar_pattern:
|
| 1704 |
if len(orig_text.split()) == 1 and len(corr_text.split()) == 1:
|
src/nlp/grammar/grammar_rules.py
CHANGED
|
@@ -350,8 +350,18 @@ class ArabicGrammarGuard:
|
|
| 350 |
# حروف الجر المتصلة بدون مسافة (بأخوك، لأبوك -> بأخيك، لأبيك)
|
| 351 |
text = re.sub(r'\b([وف]?[بل])(أبوك|أباك|أخوك|أخاك|ذو|ذا)\b',
|
| 352 |
lambda m: f"{m.group(1)}{m.group(2).replace('و', 'ي').replace('ا', 'ي')}", text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 353 |
return text
|
| 354 |
|
|
|
|
| 355 |
def process(self, original_text, generated_text):
|
| 356 |
"""Apply all grammar rules to model output."""
|
| 357 |
text = self.preserve_numbers(original_text, generated_text)
|
|
|
|
| 350 |
# حروف الجر المتصلة بدون مسافة (بأخوك، لأبوك -> بأخيك، لأبيك)
|
| 351 |
text = re.sub(r'\b([وف]?[بل])(أبوك|أباك|أخوك|أخاك|ذو|ذا)\b',
|
| 352 |
lambda m: f"{m.group(1)}{m.group(2).replace('و', 'ي').replace('ا', 'ي')}", text)
|
| 353 |
+
|
| 354 |
+
# ── NEW: Preposition + ون→ين case (في المهندسون→المهندسين) ──
|
| 355 |
+
text = re.sub(r'\b([وف]?(?:في|من|إلى|على|عن|حتى))\s+([أ-ي]{4,})(ون|ان)\b', r'\1 \2ين', text)
|
| 356 |
+
text = re.sub(r'\b([وف]?[بلكف])ال([أ-ي]{4,})(ون|ان)\b', r'\1ال\2ين', text)
|
| 357 |
+
text = re.sub(r'\b([وف]?ل)([أ-ي]{4,})(ون|ان)\b', r'\1\2ين', text)
|
| 358 |
+
|
| 359 |
+
# ── NEW: Nasb/Jazm ون→وا (لن يذهبون→يذهبوا) ──
|
| 360 |
+
text = re.sub(r'\b(لن|لم|كي|لكي|حتى|أن)\s+([أ-ي]+)(ون)\b', r'\1 \2وا', text)
|
| 361 |
+
|
| 362 |
return text
|
| 363 |
|
| 364 |
+
|
| 365 |
def process(self, original_text, generated_text):
|
| 366 |
"""Apply all grammar rules to model output."""
|
| 367 |
text = self.preserve_numbers(original_text, generated_text)
|
tests/phase10/reports/phase10_results.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|