Commit ·
b9b6999
1
Parent(s): dd35746
FIX-48: Dedicated ه→ة pass for IV words (الحكومه→الحكومة)
Browse filesOOV cleanup can't fix words like الحكومه and الشركه because they're IV
in the BERT vocabulary. This dedicated pass runs on ALL words (not just
OOV) and converts ه→ة when:
1. Word ends in ه preceded by a consonant
2. The ة form is also IV
3. Word is not in protected list (فيه, عليه, له, etc.)
Targets PC034 (الحكومه) and PC035 (الشركه).
Tests: 39 passing.
- src/app.py +43 -0
src/app.py
CHANGED
|
@@ -2099,6 +2099,49 @@ def analyze_text():
|
|
| 2099 |
# ── FIX-07: Religious text already detected above (before spelling) ──
|
| 2100 |
# _is_religious_text was set earlier to skip ALL stages for sacred text
|
| 2101 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2102 |
# ── FIX-03: Structured content protection ──
|
| 2103 |
# Protect URLs, emails, dates, code etc. from grammar model destruction
|
| 2104 |
_PROTECTED_PATTERNS = [
|
|
|
|
| 2099 |
# ── FIX-07: Religious text already detected above (before spelling) ──
|
| 2100 |
# _is_religious_text was set earlier to skip ALL stages for sacred text
|
| 2101 |
|
| 2102 |
+
# ── FIX-48: Dedicated ه→ة pass (runs on ALL words, not just OOV) ──
|
| 2103 |
+
# Words like الحكومه and الشركه are IV in BERT vocab, so OOV cleanup
|
| 2104 |
+
# skips them. This pass converts ه→ة when the ة form is also IV,
|
| 2105 |
+
# preferring standard orthography.
|
| 2106 |
+
if not _is_religious_text:
|
| 2107 |
+
try:
|
| 2108 |
+
from nlp.spelling.araspell_service import get_spelling_model
|
| 2109 |
+
_hata_checker = get_spelling_model()
|
| 2110 |
+
_hata_text = ctx.current_text
|
| 2111 |
+
_hata_words = _hata_text.split()
|
| 2112 |
+
_hata_changed = False
|
| 2113 |
+
_hata_result = []
|
| 2114 |
+
# Words that genuinely end in ه (not ة)
|
| 2115 |
+
_PROTECTED_HA = {
|
| 2116 |
+
'الله', 'لله', 'فيه', 'عليه', 'منه', 'به', 'له', 'إليه',
|
| 2117 |
+
'وجه', 'نزه', 'سفه', 'فقه', 'نبه', 'شبه', 'مكره', 'تنبه',
|
| 2118 |
+
'اتجه', 'توجه', 'تشابه', 'وفيه', 'وعليه', 'ومنه', 'وله',
|
| 2119 |
+
'دراسته', 'دراستها', 'حياته', 'حياتها',
|
| 2120 |
+
}
|
| 2121 |
+
_CONSONANTS = set('بتثجحخدذرزسشصضطظعغفقكلمنهوي')
|
| 2122 |
+
for _hw_idx, _hw in enumerate(_hata_words):
|
| 2123 |
+
_hw_clean = _hw.rstrip('.،؛؟!?!')
|
| 2124 |
+
if (len(_hw_clean) >= 4 and _hw_clean.endswith('ه')
|
| 2125 |
+
and _hw_clean not in _PROTECTED_HA
|
| 2126 |
+
and _hw_clean[-2] in _CONSONANTS):
|
| 2127 |
+
_ta_cand = _hw_clean[:-1] + 'ة'
|
| 2128 |
+
if _hata_checker.vocab_manager.is_iv(_ta_cand):
|
| 2129 |
+
_punct_suffix = _hw[len(_hw_clean):]
|
| 2130 |
+
logger.info(
|
| 2131 |
+
f"[HA-TA] ه→ة fix: '{_hw}'→'{_ta_cand}{_punct_suffix}'"
|
| 2132 |
+
)
|
| 2133 |
+
_hata_result.append(_ta_cand + _punct_suffix)
|
| 2134 |
+
_hata_changed = True
|
| 2135 |
+
continue
|
| 2136 |
+
_hata_result.append(_hw)
|
| 2137 |
+
if _hata_changed:
|
| 2138 |
+
_hata_new = ' '.join(_hata_result)
|
| 2139 |
+
logger.info(f"[HA-TA] Applied: '{_hata_text[:60]}' → '{_hata_new[:60]}'")
|
| 2140 |
+
ctx.mutate_text(_hata_new, OffsetMapper)
|
| 2141 |
+
current_text = ctx.current_text
|
| 2142 |
+
except Exception as e:
|
| 2143 |
+
logger.warning(f"[HA-TA] Failed: {type(e).__name__}: {e}")
|
| 2144 |
+
|
| 2145 |
# ── FIX-03: Structured content protection ──
|
| 2146 |
# Protect URLs, emails, dates, code etc. from grammar model destruction
|
| 2147 |
_PROTECTED_PATTERNS = [
|