Commit ·
638b53d
1
Parent(s): a037662
FIX-42: Multi-layer guards for spelling and grammar corruptions
Browse filesFIX-42a: Length ratio guard — reject spelling corrections that shrink word by >30%
(catches والممرضات→والرضا)
FIX-42b: First-letter change guard — block corrections that change root initial
(catches افهمه→تفهمة, واحتاج→وتحتاج)
FIX-42c: Grammar feminine ending guard — block grammar stripping ة/ه
(catches المديره→المدير)
FIX-42d: Grammar trailing letter guard — block grammar adding ا/ي
(catches واجب→واجبا, معطف→معطفا)
Tests: 39 passing.
- src/app.py +71 -0
src/app.py
CHANGED
|
@@ -833,6 +833,46 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
|
|
| 833 |
)
|
| 834 |
return 0.0
|
| 835 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 836 |
# ── GUARD 1: Numeral protection (Phase 1, BUG-011/012/E1) ──
|
| 837 |
# Reject corrections that remove/change/introduce digits.
|
| 838 |
# Numeral hallucination is a complete-replacement failure mode.
|
|
@@ -2091,6 +2131,22 @@ def analyze_text():
|
|
| 2091 |
)
|
| 2092 |
continue
|
| 2093 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2094 |
# Evaluate grammar patterns early to bypass heuristic blocks.
|
| 2095 |
_is_grammar_pattern = False
|
| 2096 |
if orig_text and corr_text:
|
|
@@ -2152,6 +2208,21 @@ def analyze_text():
|
|
| 2152 |
_is_grammar_pattern = True
|
| 2153 |
|
| 2154 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2155 |
# ── FIX-27a: Grammar structured data protection ──
|
| 2156 |
# Block grammar diffs where the original contains digits.
|
| 2157 |
# The grammar model corrupts dates/numbers/times/percentages.
|
|
|
|
| 833 |
)
|
| 834 |
return 0.0
|
| 835 |
|
| 836 |
+
# ── FIX-42a: Length ratio guard ──
|
| 837 |
+
# Block corrections that shrink the word significantly (>30% shorter).
|
| 838 |
+
# Catches: والممرضات(9)→والرضا(6), للطالبه(7)→للطالب(6), شجعتهم(6)→يجعلهم(6)
|
| 839 |
+
# These often indicate the model hallucinated a different word.
|
| 840 |
+
_orig_len = len(orig_word)
|
| 841 |
+
_corr_len = len(corr_word)
|
| 842 |
+
if _orig_len >= 5 and _corr_len < _orig_len * 0.7:
|
| 843 |
+
logger.info(
|
| 844 |
+
f"[SPELLING] Blocked length shrink: '{orig_word}'→'{corr_word}' "
|
| 845 |
+
f"(len {_orig_len}→{_corr_len}, ratio={_corr_len/_orig_len:.2f})"
|
| 846 |
+
)
|
| 847 |
+
return 0.0
|
| 848 |
+
|
| 849 |
+
# ── FIX-42b: First-letter change guard ──
|
| 850 |
+
# Block corrections that change the first character (after stripping common prefixes).
|
| 851 |
+
# Catches: افهمه→تفهمة (أ→ت), واحتاج→وتحتاج (ا→ت).
|
| 852 |
+
# The first root letter almost never changes in a typo — it's a hallucination.
|
| 853 |
+
if _orig_len >= 3 and _corr_len >= 3:
|
| 854 |
+
# Strip common prefixes (ال, و, ف, ب, ل, ك) to compare root starts
|
| 855 |
+
_PREFIXES = ('وال', 'فال', 'بال', 'كال', 'لل', 'ال', 'و', 'ف', 'ب', 'ل', 'ك')
|
| 856 |
+
_o_root = orig_word
|
| 857 |
+
_c_root = corr_word
|
| 858 |
+
for _pfx in _PREFIXES:
|
| 859 |
+
if _o_root.startswith(_pfx) and len(_o_root) > len(_pfx) + 1:
|
| 860 |
+
_o_root = _o_root[len(_pfx):]
|
| 861 |
+
break
|
| 862 |
+
for _pfx in _PREFIXES:
|
| 863 |
+
if _c_root.startswith(_pfx) and len(_c_root) > len(_pfx) + 1:
|
| 864 |
+
_c_root = _c_root[len(_pfx):]
|
| 865 |
+
break
|
| 866 |
+
# If roots start with different letters AND this isn't an orthographic pair
|
| 867 |
+
_HAMZA_CHARS = set('أإآاء')
|
| 868 |
+
if (_o_root and _c_root and _o_root[0] != _c_root[0]
|
| 869 |
+
and not (_o_root[0] in _HAMZA_CHARS and _c_root[0] in _HAMZA_CHARS)):
|
| 870 |
+
logger.info(
|
| 871 |
+
f"[SPELLING] Blocked first-letter change: '{orig_word}'→'{corr_word}' "
|
| 872 |
+
f"(root '{_o_root[0]}'→'{_c_root[0]}')"
|
| 873 |
+
)
|
| 874 |
+
return 0.0
|
| 875 |
+
|
| 876 |
# ── GUARD 1: Numeral protection (Phase 1, BUG-011/012/E1) ──
|
| 877 |
# Reject corrections that remove/change/introduce digits.
|
| 878 |
# Numeral hallucination is a complete-replacement failure mode.
|
|
|
|
| 2131 |
)
|
| 2132 |
continue
|
| 2133 |
|
| 2134 |
+
# ── FIX-42c: Grammar ة stripping guard ──
|
| 2135 |
+
# Block grammar changes that remove feminine ending ة/ه.
|
| 2136 |
+
# Catches: المديره→المدير, للطالبه→للطالب
|
| 2137 |
+
if orig_text and corr_text:
|
| 2138 |
+
_o_g = orig_text.rstrip('.،؛؟!?!')
|
| 2139 |
+
_c_g = corr_text.rstrip('.،؛؟!?!')
|
| 2140 |
+
if (_o_g.endswith(('ه', 'ة')) and not _c_g.endswith(('ه', 'ة'))
|
| 2141 |
+
and (_c_g == _o_g[:-1] or len(_c_g) < len(_o_g))):
|
| 2142 |
+
logger.info(
|
| 2143 |
+
f"[GRAMMAR] Blocked feminine ending strip: "
|
| 2144 |
+
f"'{orig_text}'→'{corr_text}'"
|
| 2145 |
+
)
|
| 2146 |
+
continue
|
| 2147 |
+
|
| 2148 |
+
|
| 2149 |
+
|
| 2150 |
# Evaluate grammar patterns early to bypass heuristic blocks.
|
| 2151 |
_is_grammar_pattern = False
|
| 2152 |
if orig_text and corr_text:
|
|
|
|
| 2208 |
_is_grammar_pattern = True
|
| 2209 |
|
| 2210 |
|
| 2211 |
+
# ── FIX-42d: Grammar trailing letter addition guard ──
|
| 2212 |
+
# Block grammar changes that add ا/ي to end of IV words.
|
| 2213 |
+
# Catches: واجب→واجبا, معطف→معطفا
|
| 2214 |
+
# Must come AFTER _is_grammar_pattern so we don't block valid grammar.
|
| 2215 |
+
if not _is_grammar_pattern and orig_text and corr_text:
|
| 2216 |
+
_o_g2 = orig_text.rstrip('.،؛؟!?!')
|
| 2217 |
+
_c_g2 = corr_text.rstrip('.،؛؟!?!')
|
| 2218 |
+
if (len(_c_g2) == len(_o_g2) + 1 and _c_g2.startswith(_o_g2)
|
| 2219 |
+
and _c_g2[-1] in ('ا', 'ي')):
|
| 2220 |
+
logger.info(
|
| 2221 |
+
f"[GRAMMAR] Blocked trailing letter addition: "
|
| 2222 |
+
f"'{orig_text}'→'{corr_text}'"
|
| 2223 |
+
)
|
| 2224 |
+
continue
|
| 2225 |
+
|
| 2226 |
# ── FIX-27a: Grammar structured data protection ──
|
| 2227 |
# Block grammar diffs where the original contains digits.
|
| 2228 |
# The grammar model corrupts dates/numbers/times/percentages.
|