youssefreda9 commited on
Commit
47dc3d5
·
1 Parent(s): 180a6a0

FIX-46: Expanded OOV cleanup with 4 new patterns

Browse files

FIX-46a: ه→ة fix (الحكومه→الحكومة, الشركه→الشركة) — PC034, PC035
FIX-46b: Expanded trailing و removal (any char before و, not just
restricted set) — PC004
FIX-46c: Trailing و→وا for verbs (حضرو→حضروا) — PC042
FIX-46d: Handle .و artifact (الدروس.و→الدروس.) — PC010

All changes are IV-validated (only applied when result is in vocabulary).
Expected improvement: +4-6 cases.
Tests: 39 passing.

Files changed (1) hide show
  1. src/app.py +59 -21
src/app.py CHANGED
@@ -2010,20 +2010,37 @@ def analyze_text():
2010
  _oov_result.append(_ow)
2011
  continue
2012
 
2013
- # ── Trailing و removal (from legacy AraSpell L263-267) ──
2014
- # الماضيةو → الماضية, المصنعو → المصنع, الدروسو → الدروس
2015
- if (len(_ow_clean) > 4 and _ow_clean.endswith('و')
2016
- and _ow_clean[-2] in 'ةهاأإآءين'):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2017
  _wo_cand = _ow_clean[:-1]
2018
  if _oov_checker.vocab_manager.is_iv(_wo_cand):
2019
- _punct_suffix = _ow[len(_ow_clean):] # preserve punctuation
2020
  logger.info(
2021
  f"[OOV-CLEANUP] Trailing و fix: '{_ow}'→'{_wo_cand}{_punct_suffix}'"
2022
  )
2023
  _oov_result.append(_wo_cand + _punct_suffix)
2024
  _oov_changed = True
2025
-
2026
- # Create a patch for the UI
2027
  _ow_pos = sum(len(w) + 1 for w in _oov_words[:_ow_idx])
2028
  if _ow_pos + len(_ow) <= len(_oov_text):
2029
  ctx.add_patch(
@@ -2032,20 +2049,41 @@ def analyze_text():
2032
  )
2033
  continue
2034
 
2035
- # ── Edit-distance-1 OOV→IV correction ──
2036
- # DISABLED: Caused 3 regressions (PC037, PC047, PC049) with 0 gains.
2037
- # Problem: Arabic has many valid OOV conjugated verbs (ادرسي)
2038
- # whose edit-1 IV neighbor (ادريس) is a completely different word.
2039
- # The frequency-based selection picks the wrong candidate.
2040
- # TODO: Re-enable with better candidate filtering (e.g. morphological
2041
- # compatibility check, same-POS requirement).
2042
- # try:
2043
- # _ed1_candidates = _oov_checker.edit_corrector.known(
2044
- # _oov_checker.edit_corrector.edits1(_ow_clean)
2045
- # )
2046
- # ...
2047
- # except Exception:
2048
- # pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2049
 
2050
  _oov_result.append(_ow)
2051
 
 
2010
  _oov_result.append(_ow)
2011
  continue
2012
 
2013
+ _punct_suffix = _ow[len(_ow_clean):] # preserve punctuation
2014
+
2015
+ # ── FIX-46a: ه→ة fix (vocab-validated) ──
2016
+ # الحكومه→الحكومة, الشركه→الشركة, المدرسه→المدرسة
2017
+ if len(_ow_clean) >= 4 and _ow_clean.endswith('ه'):
2018
+ _ta_cand = _ow_clean[:-1] + 'ة'
2019
+ if _oov_checker.vocab_manager.is_iv(_ta_cand):
2020
+ logger.info(
2021
+ f"[OOV-CLEANUP] ه→ة fix: '{_ow}'→'{_ta_cand}{_punct_suffix}'"
2022
+ )
2023
+ _oov_result.append(_ta_cand + _punct_suffix)
2024
+ _oov_changed = True
2025
+ _ow_pos = sum(len(w) + 1 for w in _oov_words[:_ow_idx])
2026
+ if _ow_pos + len(_ow) <= len(_oov_text):
2027
+ ctx.add_patch(
2028
+ 'spelling', _ow_pos, _ow_pos + len(_ow),
2029
+ _ta_cand + _punct_suffix, confidence=0.8,
2030
+ )
2031
+ continue
2032
+
2033
+ # ── FIX-46b: Trailing و removal (expanded) ──
2034
+ # المصنعو→المصنع, الماضيةو→الماضية
2035
+ # Expanded char set: ANY Arabic letter before و (if result is IV)
2036
+ if len(_ow_clean) > 4 and _ow_clean.endswith('و'):
2037
  _wo_cand = _ow_clean[:-1]
2038
  if _oov_checker.vocab_manager.is_iv(_wo_cand):
 
2039
  logger.info(
2040
  f"[OOV-CLEANUP] Trailing و fix: '{_ow}'→'{_wo_cand}{_punct_suffix}'"
2041
  )
2042
  _oov_result.append(_wo_cand + _punct_suffix)
2043
  _oov_changed = True
 
 
2044
  _ow_pos = sum(len(w) + 1 for w in _oov_words[:_ow_idx])
2045
  if _ow_pos + len(_ow) <= len(_oov_text):
2046
  ctx.add_patch(
 
2049
  )
2050
  continue
2051
 
2052
+ # ── FIX-46c: Trailing و→وا for verbs ──
2053
+ # حضرو→حضروا, صممو→صمموا (missing alif)
2054
+ _woa_cand = _ow_clean + 'ا'
2055
+ if _oov_checker.vocab_manager.is_iv(_woa_cand):
2056
+ logger.info(
2057
+ f"[OOV-CLEANUP] و→وا fix: '{_ow}'→'{_woa_cand}{_punct_suffix}'"
2058
+ )
2059
+ _oov_result.append(_woa_cand + _punct_suffix)
2060
+ _oov_changed = True
2061
+ _ow_pos = sum(len(w) + 1 for w in _oov_words[:_ow_idx])
2062
+ if _ow_pos + len(_ow) <= len(_oov_text):
2063
+ ctx.add_patch(
2064
+ 'spelling', _ow_pos, _ow_pos + len(_ow),
2065
+ _woa_cand + _punct_suffix, confidence=0.7,
2066
+ )
2067
+ continue
2068
+
2069
+ # ── FIX-46d: Handle .و pattern ──
2070
+ # الدروس.و→الدروس (period + و artifact)
2071
+ if _ow.endswith('.و') or _ow.endswith('،و'):
2072
+ _dotwo_cand = _ow[:-2] # remove both . and و
2073
+ _dotwo_clean = _dotwo_cand.rstrip('.،؛؟!?!')
2074
+ if len(_dotwo_clean) >= 3 and _oov_checker.vocab_manager.is_iv(_dotwo_clean):
2075
+ logger.info(
2076
+ f"[OOV-CLEANUP] .و artifact fix: '{_ow}'→'{_dotwo_clean}.'"
2077
+ )
2078
+ _oov_result.append(_dotwo_clean + '.')
2079
+ _oov_changed = True
2080
+ _ow_pos = sum(len(w) + 1 for w in _oov_words[:_ow_idx])
2081
+ if _ow_pos + len(_ow) <= len(_oov_text):
2082
+ ctx.add_patch(
2083
+ 'spelling', _ow_pos, _ow_pos + len(_ow),
2084
+ _dotwo_clean + '.', confidence=0.75,
2085
+ )
2086
+ continue
2087
 
2088
  _oov_result.append(_ow)
2089