youssefreda9 commited on
Commit
5369041
·
1 Parent(s): 79db738

FIX-41: Block spelling corrections that add trailing ا/ي/و to IV words

Browse files

Prevents model corruptions like واجب→واجبا, معطف→معطفا where the model
inappropriately adds accusative markers to already-correct words.
Tests: 39 passing.

Files changed (1) hide show
  1. src/app.py +12 -0
src/app.py CHANGED
@@ -899,6 +899,18 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
899
  if corr_word == orig_word[:-1] or len(corr_word) < len(orig_word):
900
  return 0.0
901
 
 
 
 
 
 
 
 
 
 
 
 
 
902
  # CRITICAL: If both words are valid Arabic words, only accept known fixes.
903
  # This prevents the spelling model from changing one correct word to another
904
  # (e.g. وكان→وكأن, which changes "and was" to "as if" — a meaning change).
 
899
  if corr_word == orig_word[:-1] or len(corr_word) < len(orig_word):
900
  return 0.0
901
 
902
+ # ── FIX-41: Block corrections that ADD trailing ا/ي to IV words ──
903
+ # Model sometimes adds accusative markers: واجب→واجبا, معطف→معطفا.
904
+ # If the original word is IV and the correction just appends a letter, reject.
905
+ if vocab_manager and len(corr_word) == len(orig_word) + 1 and corr_word.startswith(orig_word):
906
+ _appended_char = corr_word[-1]
907
+ if _appended_char in ('ا', 'ي', 'و') and vocab_manager.is_iv(orig_word):
908
+ logger.info(
909
+ f"[SPELLING] Blocked trailing '{_appended_char}' addition: "
910
+ f"'{orig_word}'→'{corr_word}' (original is IV)"
911
+ )
912
+ return 0.0
913
+
914
  # CRITICAL: If both words are valid Arabic words, only accept known fixes.
915
  # This prevents the spelling model from changing one correct word to another
916
  # (e.g. وكان→وكأن, which changes "and was" to "as if" — a meaning change).