Commit ·
180a6a0
1
Parent(s): e38dee9
Disable edit-1 OOV cleanup (caused 3 regressions)
Browse filesThe edit-distance-1 OOV→IV cleanup in FIX-44 caused PC037, PC047, PC049
to regress with zero improvements. Root cause: Arabic conjugated verbs
(ادرسي) are OOV but their edit-1 IV neighbor (ادريس) is a completely
different word. Frequency-based selection picks wrong candidates.
Trailing و removal stays active (no regressions from that).
Tests: 39 passing.
- src/app.py +13 -31
src/app.py
CHANGED
|
@@ -2033,37 +2033,19 @@ def analyze_text():
|
|
| 2033 |
continue
|
| 2034 |
|
| 2035 |
# ── Edit-distance-1 OOV→IV correction ──
|
| 2036 |
-
#
|
| 2037 |
-
|
| 2038 |
-
|
| 2039 |
-
|
| 2040 |
-
|
| 2041 |
-
|
| 2042 |
-
|
| 2043 |
-
|
| 2044 |
-
|
| 2045 |
-
|
| 2046 |
-
|
| 2047 |
-
|
| 2048 |
-
|
| 2049 |
-
_best_cand[0] in 'أإآاء' and _ow_clean[0] in 'أإآاء'
|
| 2050 |
-
):
|
| 2051 |
-
_punct_suffix = _ow[len(_ow_clean):]
|
| 2052 |
-
logger.info(
|
| 2053 |
-
f"[OOV-CLEANUP] Edit-1 fix: '{_ow}'→'{_best_cand}{_punct_suffix}'"
|
| 2054 |
-
)
|
| 2055 |
-
_oov_result.append(_best_cand + _punct_suffix)
|
| 2056 |
-
_oov_changed = True
|
| 2057 |
-
|
| 2058 |
-
_ow_pos = sum(len(w) + 1 for w in _oov_words[:_ow_idx])
|
| 2059 |
-
if _ow_pos + len(_ow) <= len(_oov_text):
|
| 2060 |
-
ctx.add_patch(
|
| 2061 |
-
'spelling', _ow_pos, _ow_pos + len(_ow),
|
| 2062 |
-
_best_cand + _punct_suffix, confidence=0.65,
|
| 2063 |
-
)
|
| 2064 |
-
continue
|
| 2065 |
-
except Exception:
|
| 2066 |
-
pass # Edit-distance fallback is best-effort
|
| 2067 |
|
| 2068 |
_oov_result.append(_ow)
|
| 2069 |
|
|
|
|
| 2033 |
continue
|
| 2034 |
|
| 2035 |
# ── Edit-distance-1 OOV→IV correction ──
|
| 2036 |
+
# DISABLED: Caused 3 regressions (PC037, PC047, PC049) with 0 gains.
|
| 2037 |
+
# Problem: Arabic has many valid OOV conjugated verbs (ادرسي)
|
| 2038 |
+
# whose edit-1 IV neighbor (ادريس) is a completely different word.
|
| 2039 |
+
# The frequency-based selection picks the wrong candidate.
|
| 2040 |
+
# TODO: Re-enable with better candidate filtering (e.g. morphological
|
| 2041 |
+
# compatibility check, same-POS requirement).
|
| 2042 |
+
# try:
|
| 2043 |
+
# _ed1_candidates = _oov_checker.edit_corrector.known(
|
| 2044 |
+
# _oov_checker.edit_corrector.edits1(_ow_clean)
|
| 2045 |
+
# )
|
| 2046 |
+
# ...
|
| 2047 |
+
# except Exception:
|
| 2048 |
+
# pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2049 |
|
| 2050 |
_oov_result.append(_ow)
|
| 2051 |
|