Commit ·
5369041
1
Parent(s): 79db738
FIX-41: Block spelling corrections that add trailing ا/ي/و to IV words
Browse filesPrevents model corruptions like واجب→واجبا, معطف→معطفا where the model
inappropriately adds accusative markers to already-correct words.
Tests: 39 passing.
- src/app.py +12 -0
src/app.py
CHANGED
|
@@ -899,6 +899,18 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
|
|
| 899 |
if corr_word == orig_word[:-1] or len(corr_word) < len(orig_word):
|
| 900 |
return 0.0
|
| 901 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 902 |
# CRITICAL: If both words are valid Arabic words, only accept known fixes.
|
| 903 |
# This prevents the spelling model from changing one correct word to another
|
| 904 |
# (e.g. وكان→وكأن, which changes "and was" to "as if" — a meaning change).
|
|
|
|
| 899 |
if corr_word == orig_word[:-1] or len(corr_word) < len(orig_word):
|
| 900 |
return 0.0
|
| 901 |
|
| 902 |
+
# ── FIX-41: Block corrections that ADD trailing ا/ي to IV words ──
|
| 903 |
+
# Model sometimes adds accusative markers: واجب→واجبا, معطف→معطفا.
|
| 904 |
+
# If the original word is IV and the correction just appends a letter, reject.
|
| 905 |
+
if vocab_manager and len(corr_word) == len(orig_word) + 1 and corr_word.startswith(orig_word):
|
| 906 |
+
_appended_char = corr_word[-1]
|
| 907 |
+
if _appended_char in ('ا', 'ي', 'و') and vocab_manager.is_iv(orig_word):
|
| 908 |
+
logger.info(
|
| 909 |
+
f"[SPELLING] Blocked trailing '{_appended_char}' addition: "
|
| 910 |
+
f"'{orig_word}'→'{corr_word}' (original is IV)"
|
| 911 |
+
)
|
| 912 |
+
return 0.0
|
| 913 |
+
|
| 914 |
# CRITICAL: If both words are valid Arabic words, only accept known fixes.
|
| 915 |
# This prevents the spelling model from changing one correct word to another
|
| 916 |
# (e.g. وكان→وكأن, which changes "and was" to "as if" — a meaning change).
|