File size: 2,709 Bytes
31f02a4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | """
Phase 12 (B3) — Test diacritic normalization before IVtoOOV validation.
Verifies that grammar corrections with diacritics (e.g. يفعلوَ) are not
rejected by the IVtoOOV filter, since the diacritic-stripped form (يفعلوا)
is a valid in-vocabulary word.
"""
import re
import sys
import os
# Add src to path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
def test_diacritic_stripping():
"""Test that Arabic diacritics are properly stripped."""
DIACRITICS_RE = re.compile(r'[\u064B-\u065F\u0670]')
cases = [
('يفعلوَ', 'يفعلو'), # fatha at end
('لعبوَ', 'لعبو'), # fatha at end
('كَتَبَ', 'كتب'), # multiple fatha
('مُعَلِّم', 'معلم'), # damma + fatha + kasra + shadda — all stripped
('طالبٌ', 'طالب'), # tanween damma
('كتاباً', 'كتابا'), # tanween fatha
('بسمِ', 'بسم'), # kasra
]
for input_text, expected in cases:
result = DIACRITICS_RE.sub('', input_text)
assert result == expected, (
f"Diacritic stripping failed: '{input_text}' → '{result}' "
f"(expected '{expected}')"
)
print(f" ✅ '{input_text}' → '{result}'")
def test_ivtooov_with_diacritics():
"""Test that IVtoOOV check strips diacritics before validation."""
try:
from nlp.spelling.araspell_service import get_spelling_model
vm = get_spelling_model().vocab_manager
if not vm:
print(" ⚠️ VocabularyManager not available — skipping")
return
DIACRITICS_RE = re.compile(r'[\u064B-\u065F\u0670]')
# Test cases: (diacriticed_form, should_be_iv_after_stripping)
cases = [
('يفعلوَ', True), # يفعلو → should check if IV
('لعبوَ', True), # لعبو → should check if IV
('حضروا', True), # No diacritics, should be IV
('يذهبون', True), # No diacritics, should be IV
]
for word, _ in cases:
clean = DIACRITICS_RE.sub('', word)
is_iv = vm.is_iv(clean)
print(f" {'✅' if is_iv else '⚠️'} '{word}' → '{clean}' IV={is_iv}")
except ImportError:
print(" ⚠️ Cannot import spelling model — skipping (expected in test env)")
if __name__ == '__main__':
print("Test: Diacritic Stripping")
test_diacritic_stripping()
print("\nTest: IVtoOOV with Diacritics")
test_ivtooov_with_diacritics()
print("\n✅ All diacritic normalization tests passed")
|