Commit ·
e0bf168
1
Parent(s): bd94153
fix(spelling): strict ortho-only filter prevents word corruption
Browse filesBEFORE: سعراته→سعادتة ACCEPTED (root change, meaning change!)
عضلية→عملية ACCEPTED (consonant change, different word!)
AFTER: Only orthographic swaps allowed for OOV words:
ه↔ة, ا↔أ↔إ↔آ, ي↔ى, ء↔أ↔ؤ↔ئ, ؤ↔و, ئ↔ي
10/10 test cases pass. 59/59 hardening tests pass.
- src/app.py +29 -3
src/app.py
CHANGED
|
@@ -814,9 +814,35 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
|
|
| 814 |
dist = _levenshtein(orig_word, corr_word)
|
| 815 |
max_len = max(len(orig_word), len(corr_word))
|
| 816 |
|
| 817 |
-
#
|
| 818 |
-
#
|
| 819 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 820 |
|
| 821 |
|
| 822 |
def _is_spelling_only_change(original: str, correction: str) -> bool:
|
|
|
|
| 814 |
dist = _levenshtein(orig_word, corr_word)
|
| 815 |
max_len = max(len(orig_word), len(corr_word))
|
| 816 |
|
| 817 |
+
# Tighter filter for OOV words: reject edits that change word roots
|
| 818 |
+
# Allow max 2 edits at max 50% of word length
|
| 819 |
+
if dist > 2 or (dist / max_len) > 0.5:
|
| 820 |
+
return False
|
| 821 |
+
|
| 822 |
+
# CRITICAL: Only allow ORTHOGRAPHIC fixes (ه↔ة, ا↔أ↔إ↔آ, ي↔ى).
|
| 823 |
+
# Any other letter change means the word's ROOT is different
|
| 824 |
+
# (e.g. عضلية→عملية ض→م = completely different word!)
|
| 825 |
+
ORTHO_PAIRS = {
|
| 826 |
+
('ه', 'ة'), ('ة', 'ه'),
|
| 827 |
+
('ا', 'أ'), ('أ', 'ا'), ('ا', 'إ'), ('إ', 'ا'), ('ا', 'آ'), ('آ', 'ا'),
|
| 828 |
+
('ي', 'ى'), ('ى', 'ي'),
|
| 829 |
+
('ؤ', 'و'), ('و', 'ؤ'), # hamza on waw
|
| 830 |
+
('ئ', 'ي'), ('ي', 'ئ'), # hamza on ya
|
| 831 |
+
('ء', 'أ'), ('أ', 'ء'), # standalone hamza ↔ hamza on alef
|
| 832 |
+
('ء', 'ؤ'), ('ؤ', 'ء'), # standalone hamza ↔ hamza on waw
|
| 833 |
+
('ء', 'ئ'), ('ئ', 'ء'), # standalone hamza ↔ hamza on ya
|
| 834 |
+
}
|
| 835 |
+
# Check every character pair — reject if ANY non-orthographic change
|
| 836 |
+
if len(orig_word) != len(corr_word):
|
| 837 |
+
# Length change = structural change, not just orthographic
|
| 838 |
+
# Exception: if diff is just adding/removing ا at start (hamza)
|
| 839 |
+
if abs(len(orig_word) - len(corr_word)) > 1:
|
| 840 |
+
return False
|
| 841 |
+
for a, b in zip(orig_word, corr_word):
|
| 842 |
+
if a != b and (a, b) not in ORTHO_PAIRS:
|
| 843 |
+
return False
|
| 844 |
+
|
| 845 |
+
return True
|
| 846 |
|
| 847 |
|
| 848 |
def _is_spelling_only_change(original: str, correction: str) -> bool:
|