youssefreda9 commited on
Commit
b9b6999
·
1 Parent(s): dd35746

FIX-48: Dedicated ه→ة pass for IV words (الحكومه→الحكومة)

Browse files

OOV cleanup can't fix words like الحكومه and الشركه because they're IV
in the BERT vocabulary. This dedicated pass runs on ALL words (not just
OOV) and converts ه→ة when:
1. Word ends in ه preceded by a consonant
2. The ة form is also IV
3. Word is not in protected list (فيه, عليه, له, etc.)

Targets PC034 (الحكومه) and PC035 (الشركه).
Tests: 39 passing.

Files changed (1) hide show
  1. src/app.py +43 -0
src/app.py CHANGED
@@ -2099,6 +2099,49 @@ def analyze_text():
2099
  # ── FIX-07: Religious text already detected above (before spelling) ──
2100
  # _is_religious_text was set earlier to skip ALL stages for sacred text
2101
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2102
  # ── FIX-03: Structured content protection ──
2103
  # Protect URLs, emails, dates, code etc. from grammar model destruction
2104
  _PROTECTED_PATTERNS = [
 
2099
  # ── FIX-07: Religious text already detected above (before spelling) ──
2100
  # _is_religious_text was set earlier to skip ALL stages for sacred text
2101
 
2102
+ # ── FIX-48: Dedicated ه→ة pass (runs on ALL words, not just OOV) ──
2103
+ # Words like الحكومه and الشركه are IV in BERT vocab, so OOV cleanup
2104
+ # skips them. This pass converts ه→ة when the ة form is also IV,
2105
+ # preferring standard orthography.
2106
+ if not _is_religious_text:
2107
+ try:
2108
+ from nlp.spelling.araspell_service import get_spelling_model
2109
+ _hata_checker = get_spelling_model()
2110
+ _hata_text = ctx.current_text
2111
+ _hata_words = _hata_text.split()
2112
+ _hata_changed = False
2113
+ _hata_result = []
2114
+ # Words that genuinely end in ه (not ة)
2115
+ _PROTECTED_HA = {
2116
+ 'الله', 'لله', 'فيه', 'عليه', 'منه', 'به', 'له', 'إليه',
2117
+ 'وجه', 'نزه', 'سفه', 'فقه', 'نبه', 'شبه', 'مكره', 'تنبه',
2118
+ 'اتجه', 'توجه', 'تشابه', 'وفيه', 'وعليه', 'ومنه', 'وله',
2119
+ 'دراسته', 'دراستها', 'حياته', 'حياتها',
2120
+ }
2121
+ _CONSONANTS = set('بتثجحخدذرزسشصضطظعغفقكلمنهوي')
2122
+ for _hw_idx, _hw in enumerate(_hata_words):
2123
+ _hw_clean = _hw.rstrip('.،؛؟!?!')
2124
+ if (len(_hw_clean) >= 4 and _hw_clean.endswith('ه')
2125
+ and _hw_clean not in _PROTECTED_HA
2126
+ and _hw_clean[-2] in _CONSONANTS):
2127
+ _ta_cand = _hw_clean[:-1] + 'ة'
2128
+ if _hata_checker.vocab_manager.is_iv(_ta_cand):
2129
+ _punct_suffix = _hw[len(_hw_clean):]
2130
+ logger.info(
2131
+ f"[HA-TA] ه→ة fix: '{_hw}'→'{_ta_cand}{_punct_suffix}'"
2132
+ )
2133
+ _hata_result.append(_ta_cand + _punct_suffix)
2134
+ _hata_changed = True
2135
+ continue
2136
+ _hata_result.append(_hw)
2137
+ if _hata_changed:
2138
+ _hata_new = ' '.join(_hata_result)
2139
+ logger.info(f"[HA-TA] Applied: '{_hata_text[:60]}' → '{_hata_new[:60]}'")
2140
+ ctx.mutate_text(_hata_new, OffsetMapper)
2141
+ current_text = ctx.current_text
2142
+ except Exception as e:
2143
+ logger.warning(f"[HA-TA] Failed: {type(e).__name__}: {e}")
2144
+
2145
  # ── FIX-03: Structured content protection ──
2146
  # Protect URLs, emails, dates, code etc. from grammar model destruction
2147
  _PROTECTED_PATTERNS = [