Commit ·
47dc3d5
1
Parent(s): 180a6a0
FIX-46: Expanded OOV cleanup with 4 new patterns
Browse filesFIX-46a: ه→ة fix (الحكومه→الحكومة, الشركه→الشركة) — PC034, PC035
FIX-46b: Expanded trailing و removal (any char before و, not just
restricted set) — PC004
FIX-46c: Trailing و→وا for verbs (حضرو→حضروا) — PC042
FIX-46d: Handle .و artifact (الدروس.و→الدروس.) — PC010
All changes are IV-validated (only applied when result is in vocabulary).
Expected improvement: +4-6 cases.
Tests: 39 passing.
- src/app.py +59 -21
src/app.py
CHANGED
|
@@ -2010,20 +2010,37 @@ def analyze_text():
|
|
| 2010 |
_oov_result.append(_ow)
|
| 2011 |
continue
|
| 2012 |
|
| 2013 |
-
|
| 2014 |
-
|
| 2015 |
-
|
| 2016 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2017 |
_wo_cand = _ow_clean[:-1]
|
| 2018 |
if _oov_checker.vocab_manager.is_iv(_wo_cand):
|
| 2019 |
-
_punct_suffix = _ow[len(_ow_clean):] # preserve punctuation
|
| 2020 |
logger.info(
|
| 2021 |
f"[OOV-CLEANUP] Trailing و fix: '{_ow}'→'{_wo_cand}{_punct_suffix}'"
|
| 2022 |
)
|
| 2023 |
_oov_result.append(_wo_cand + _punct_suffix)
|
| 2024 |
_oov_changed = True
|
| 2025 |
-
|
| 2026 |
-
# Create a patch for the UI
|
| 2027 |
_ow_pos = sum(len(w) + 1 for w in _oov_words[:_ow_idx])
|
| 2028 |
if _ow_pos + len(_ow) <= len(_oov_text):
|
| 2029 |
ctx.add_patch(
|
|
@@ -2032,20 +2049,41 @@ def analyze_text():
|
|
| 2032 |
)
|
| 2033 |
continue
|
| 2034 |
|
| 2035 |
-
|
| 2036 |
-
|
| 2037 |
-
|
| 2038 |
-
|
| 2039 |
-
|
| 2040 |
-
|
| 2041 |
-
|
| 2042 |
-
|
| 2043 |
-
|
| 2044 |
-
|
| 2045 |
-
|
| 2046 |
-
|
| 2047 |
-
|
| 2048 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2049 |
|
| 2050 |
_oov_result.append(_ow)
|
| 2051 |
|
|
|
|
| 2010 |
_oov_result.append(_ow)
|
| 2011 |
continue
|
| 2012 |
|
| 2013 |
+
_punct_suffix = _ow[len(_ow_clean):] # preserve punctuation
|
| 2014 |
+
|
| 2015 |
+
# ── FIX-46a: ه→ة fix (vocab-validated) ──
|
| 2016 |
+
# الحكومه→الحكومة, الشركه→الشركة, المدرسه→المدرسة
|
| 2017 |
+
if len(_ow_clean) >= 4 and _ow_clean.endswith('ه'):
|
| 2018 |
+
_ta_cand = _ow_clean[:-1] + 'ة'
|
| 2019 |
+
if _oov_checker.vocab_manager.is_iv(_ta_cand):
|
| 2020 |
+
logger.info(
|
| 2021 |
+
f"[OOV-CLEANUP] ه→ة fix: '{_ow}'→'{_ta_cand}{_punct_suffix}'"
|
| 2022 |
+
)
|
| 2023 |
+
_oov_result.append(_ta_cand + _punct_suffix)
|
| 2024 |
+
_oov_changed = True
|
| 2025 |
+
_ow_pos = sum(len(w) + 1 for w in _oov_words[:_ow_idx])
|
| 2026 |
+
if _ow_pos + len(_ow) <= len(_oov_text):
|
| 2027 |
+
ctx.add_patch(
|
| 2028 |
+
'spelling', _ow_pos, _ow_pos + len(_ow),
|
| 2029 |
+
_ta_cand + _punct_suffix, confidence=0.8,
|
| 2030 |
+
)
|
| 2031 |
+
continue
|
| 2032 |
+
|
| 2033 |
+
# ── FIX-46b: Trailing و removal (expanded) ──
|
| 2034 |
+
# المصنعو→المصنع, الماضيةو→الماضية
|
| 2035 |
+
# Expanded char set: ANY Arabic letter before و (if result is IV)
|
| 2036 |
+
if len(_ow_clean) > 4 and _ow_clean.endswith('و'):
|
| 2037 |
_wo_cand = _ow_clean[:-1]
|
| 2038 |
if _oov_checker.vocab_manager.is_iv(_wo_cand):
|
|
|
|
| 2039 |
logger.info(
|
| 2040 |
f"[OOV-CLEANUP] Trailing و fix: '{_ow}'→'{_wo_cand}{_punct_suffix}'"
|
| 2041 |
)
|
| 2042 |
_oov_result.append(_wo_cand + _punct_suffix)
|
| 2043 |
_oov_changed = True
|
|
|
|
|
|
|
| 2044 |
_ow_pos = sum(len(w) + 1 for w in _oov_words[:_ow_idx])
|
| 2045 |
if _ow_pos + len(_ow) <= len(_oov_text):
|
| 2046 |
ctx.add_patch(
|
|
|
|
| 2049 |
)
|
| 2050 |
continue
|
| 2051 |
|
| 2052 |
+
# ── FIX-46c: Trailing و→وا for verbs ──
|
| 2053 |
+
# حضرو→حضروا, صممو→صمموا (missing alif)
|
| 2054 |
+
_woa_cand = _ow_clean + 'ا'
|
| 2055 |
+
if _oov_checker.vocab_manager.is_iv(_woa_cand):
|
| 2056 |
+
logger.info(
|
| 2057 |
+
f"[OOV-CLEANUP] و→وا fix: '{_ow}'→'{_woa_cand}{_punct_suffix}'"
|
| 2058 |
+
)
|
| 2059 |
+
_oov_result.append(_woa_cand + _punct_suffix)
|
| 2060 |
+
_oov_changed = True
|
| 2061 |
+
_ow_pos = sum(len(w) + 1 for w in _oov_words[:_ow_idx])
|
| 2062 |
+
if _ow_pos + len(_ow) <= len(_oov_text):
|
| 2063 |
+
ctx.add_patch(
|
| 2064 |
+
'spelling', _ow_pos, _ow_pos + len(_ow),
|
| 2065 |
+
_woa_cand + _punct_suffix, confidence=0.7,
|
| 2066 |
+
)
|
| 2067 |
+
continue
|
| 2068 |
+
|
| 2069 |
+
# ── FIX-46d: Handle .و pattern ──
|
| 2070 |
+
# الدروس.و→الدروس (period + و artifact)
|
| 2071 |
+
if _ow.endswith('.و') or _ow.endswith('،و'):
|
| 2072 |
+
_dotwo_cand = _ow[:-2] # remove both . and و
|
| 2073 |
+
_dotwo_clean = _dotwo_cand.rstrip('.،؛؟!?!')
|
| 2074 |
+
if len(_dotwo_clean) >= 3 and _oov_checker.vocab_manager.is_iv(_dotwo_clean):
|
| 2075 |
+
logger.info(
|
| 2076 |
+
f"[OOV-CLEANUP] .و artifact fix: '{_ow}'→'{_dotwo_clean}.'"
|
| 2077 |
+
)
|
| 2078 |
+
_oov_result.append(_dotwo_clean + '.')
|
| 2079 |
+
_oov_changed = True
|
| 2080 |
+
_ow_pos = sum(len(w) + 1 for w in _oov_words[:_ow_idx])
|
| 2081 |
+
if _ow_pos + len(_ow) <= len(_oov_text):
|
| 2082 |
+
ctx.add_patch(
|
| 2083 |
+
'spelling', _ow_pos, _ow_pos + len(_ow),
|
| 2084 |
+
_dotwo_clean + '.', confidence=0.75,
|
| 2085 |
+
)
|
| 2086 |
+
continue
|
| 2087 |
|
| 2088 |
_oov_result.append(_ow)
|
| 2089 |
|