tests/test_ivtooov_diacritic_normalization.py

"""
Phase 12 (B3) — Test diacritic normalization before IVtoOOV validation.

Verifies that grammar corrections with diacritics (e.g. يفعلوَ) are not
rejected by the IVtoOOV filter, since the diacritic-stripped form (يفعلوا)
is a valid in-vocabulary word.
"""
import re
import sys
import os

# Add src to path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))


def test_diacritic_stripping():
    """Test that Arabic diacritics are properly stripped."""
    DIACRITICS_RE = re.compile(r'[\u064B-\u065F\u0670]')
    
    cases = [
        ('يفعلوَ', 'يفعلو'),      # fatha at end
        ('لعبوَ', 'لعبو'),        # fatha at end
        ('كَتَبَ', 'كتب'),        # multiple fatha
        ('مُعَلِّم', 'معلم'),      # damma + fatha + kasra + shadda — all stripped
        ('طالبٌ', 'طالب'),        # tanween damma
        ('كتاباً', 'كتابا'),      # tanween fatha
        ('بسمِ', 'بسم'),          # kasra
    ]
    
    for input_text, expected in cases:
        result = DIACRITICS_RE.sub('', input_text)
        assert result == expected, (
            f"Diacritic stripping failed: '{input_text}' → '{result}' "
            f"(expected '{expected}')"
        )
        print(f"  ✅ '{input_text}' → '{result}'")


def test_ivtooov_with_diacritics():
    """Test that IVtoOOV check strips diacritics before validation."""
    try:
        from nlp.spelling.araspell_service import get_spelling_model
        vm = get_spelling_model().vocab_manager
        if not vm:
            print("  ⚠️ VocabularyManager not available — skipping")
            return
        
        DIACRITICS_RE = re.compile(r'[\u064B-\u065F\u0670]')
        
        # Test cases: (diacriticed_form, should_be_iv_after_stripping)
        cases = [
            ('يفعلوَ', True),   # يفعلو → should check if IV
            ('لعبوَ', True),    # لعبو → should check if IV
            ('حضروا', True),   # No diacritics, should be IV
            ('يذهبون', True),  # No diacritics, should be IV
        ]
        
        for word, _ in cases:
            clean = DIACRITICS_RE.sub('', word)
            is_iv = vm.is_iv(clean)
            print(f"  {'✅' if is_iv else '⚠️'} '{word}' → '{clean}' IV={is_iv}")
            
    except ImportError:
        print("  ⚠️ Cannot import spelling model — skipping (expected in test env)")


if __name__ == '__main__':
    print("Test: Diacritic Stripping")
    test_diacritic_stripping()
    print("\nTest: IVtoOOV with Diacritics")
    test_ivtooov_with_diacritics()
    print("\n✅ All diacritic normalization tests passed")