youssefreda9 commited on
Commit
e0bf168
·
1 Parent(s): bd94153

fix(spelling): strict ortho-only filter prevents word corruption

Browse files

BEFORE: سعراته→سعادتة ACCEPTED (root change, meaning change!)
عضلية→عملية ACCEPTED (consonant change, different word!)
AFTER: Only orthographic swaps allowed for OOV words:
ه↔ة, ا↔أ↔إ↔آ, ي↔ى, ء↔أ↔ؤ↔ئ, ؤ↔و, ئ↔ي

10/10 test cases pass. 59/59 hardening tests pass.

Files changed (1) hide show
  1. src/app.py +29 -3
src/app.py CHANGED
@@ -814,9 +814,35 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
814
  dist = _levenshtein(orig_word, corr_word)
815
  max_len = max(len(orig_word), len(corr_word))
816
 
817
- # Allow at most 3 character edits and at most 50% of the word
818
- # AraSpell has its own validation pipeline, so we can be more permissive here
819
- return dist <= 3 and (dist / max_len) <= 0.5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
820
 
821
 
822
  def _is_spelling_only_change(original: str, correction: str) -> bool:
 
814
  dist = _levenshtein(orig_word, corr_word)
815
  max_len = max(len(orig_word), len(corr_word))
816
 
817
+ # Tighter filter for OOV words: reject edits that change word roots
818
+ # Allow max 2 edits at max 50% of word length
819
+ if dist > 2 or (dist / max_len) > 0.5:
820
+ return False
821
+
822
+ # CRITICAL: Only allow ORTHOGRAPHIC fixes (ه↔ة, ا↔أ↔إ↔آ, ي↔ى).
823
+ # Any other letter change means the word's ROOT is different
824
+ # (e.g. عضلية→عملية ض→م = completely different word!)
825
+ ORTHO_PAIRS = {
826
+ ('ه', 'ة'), ('ة', 'ه'),
827
+ ('ا', 'أ'), ('أ', 'ا'), ('ا', 'إ'), ('إ', 'ا'), ('ا', 'آ'), ('آ', 'ا'),
828
+ ('ي', 'ى'), ('ى', 'ي'),
829
+ ('ؤ', 'و'), ('و', 'ؤ'), # hamza on waw
830
+ ('ئ', 'ي'), ('ي', 'ئ'), # hamza on ya
831
+ ('ء', 'أ'), ('أ', 'ء'), # standalone hamza ↔ hamza on alef
832
+ ('ء', 'ؤ'), ('ؤ', 'ء'), # standalone hamza ↔ hamza on waw
833
+ ('ء', 'ئ'), ('ئ', 'ء'), # standalone hamza ↔ hamza on ya
834
+ }
835
+ # Check every character pair — reject if ANY non-orthographic change
836
+ if len(orig_word) != len(corr_word):
837
+ # Length change = structural change, not just orthographic
838
+ # Exception: if diff is just adding/removing ا at start (hamza)
839
+ if abs(len(orig_word) - len(corr_word)) > 1:
840
+ return False
841
+ for a, b in zip(orig_word, corr_word):
842
+ if a != b and (a, b) not in ORTHO_PAIRS:
843
+ return False
844
+
845
+ return True
846
 
847
 
848
  def _is_spelling_only_change(original: str, correction: str) -> bool: