youssefreda9 commited on
Commit
638b53d
·
1 Parent(s): a037662

FIX-42: Multi-layer guards for spelling and grammar corruptions

Browse files

FIX-42a: Length ratio guard — reject spelling corrections that shrink word by >30%
(catches والممرضات→والرضا)
FIX-42b: First-letter change guard — block corrections that change root initial
(catches افهمه→تفهمة, واحتاج→وتحتاج)
FIX-42c: Grammar feminine ending guard — block grammar stripping ة/ه
(catches المديره→المدير)
FIX-42d: Grammar trailing letter guard — block grammar adding ا/ي
(catches واجب→واجبا, معطف→معطفا)

Tests: 39 passing.

Files changed (1) hide show
  1. src/app.py +71 -0
src/app.py CHANGED
@@ -833,6 +833,46 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
833
  )
834
  return 0.0
835
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
836
  # ── GUARD 1: Numeral protection (Phase 1, BUG-011/012/E1) ──
837
  # Reject corrections that remove/change/introduce digits.
838
  # Numeral hallucination is a complete-replacement failure mode.
@@ -2091,6 +2131,22 @@ def analyze_text():
2091
  )
2092
  continue
2093
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2094
  # Evaluate grammar patterns early to bypass heuristic blocks.
2095
  _is_grammar_pattern = False
2096
  if orig_text and corr_text:
@@ -2152,6 +2208,21 @@ def analyze_text():
2152
  _is_grammar_pattern = True
2153
 
2154
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2155
  # ── FIX-27a: Grammar structured data protection ──
2156
  # Block grammar diffs where the original contains digits.
2157
  # The grammar model corrupts dates/numbers/times/percentages.
 
833
  )
834
  return 0.0
835
 
836
+ # ── FIX-42a: Length ratio guard ──
837
+ # Block corrections that shrink the word significantly (>30% shorter).
838
+ # Catches: والممرضات(9)→والرضا(6), للطالبه(7)→للطالب(6), شجعتهم(6)→يجعلهم(6)
839
+ # These often indicate the model hallucinated a different word.
840
+ _orig_len = len(orig_word)
841
+ _corr_len = len(corr_word)
842
+ if _orig_len >= 5 and _corr_len < _orig_len * 0.7:
843
+ logger.info(
844
+ f"[SPELLING] Blocked length shrink: '{orig_word}'→'{corr_word}' "
845
+ f"(len {_orig_len}→{_corr_len}, ratio={_corr_len/_orig_len:.2f})"
846
+ )
847
+ return 0.0
848
+
849
+ # ── FIX-42b: First-letter change guard ──
850
+ # Block corrections that change the first character (after stripping common prefixes).
851
+ # Catches: افهمه→تفهمة (أ→ت), واحتاج→وتحتاج (ا→ت).
852
+ # The first root letter almost never changes in a typo — it's a hallucination.
853
+ if _orig_len >= 3 and _corr_len >= 3:
854
+ # Strip common prefixes (ال, و, ف, ب, ل, ك) to compare root starts
855
+ _PREFIXES = ('وال', 'فال', 'بال', 'كال', 'لل', 'ال', 'و', 'ف', 'ب', 'ل', 'ك')
856
+ _o_root = orig_word
857
+ _c_root = corr_word
858
+ for _pfx in _PREFIXES:
859
+ if _o_root.startswith(_pfx) and len(_o_root) > len(_pfx) + 1:
860
+ _o_root = _o_root[len(_pfx):]
861
+ break
862
+ for _pfx in _PREFIXES:
863
+ if _c_root.startswith(_pfx) and len(_c_root) > len(_pfx) + 1:
864
+ _c_root = _c_root[len(_pfx):]
865
+ break
866
+ # If roots start with different letters AND this isn't an orthographic pair
867
+ _HAMZA_CHARS = set('أإآاء')
868
+ if (_o_root and _c_root and _o_root[0] != _c_root[0]
869
+ and not (_o_root[0] in _HAMZA_CHARS and _c_root[0] in _HAMZA_CHARS)):
870
+ logger.info(
871
+ f"[SPELLING] Blocked first-letter change: '{orig_word}'→'{corr_word}' "
872
+ f"(root '{_o_root[0]}'→'{_c_root[0]}')"
873
+ )
874
+ return 0.0
875
+
876
  # ── GUARD 1: Numeral protection (Phase 1, BUG-011/012/E1) ──
877
  # Reject corrections that remove/change/introduce digits.
878
  # Numeral hallucination is a complete-replacement failure mode.
 
2131
  )
2132
  continue
2133
 
2134
+ # ── FIX-42c: Grammar ة stripping guard ──
2135
+ # Block grammar changes that remove feminine ending ة/ه.
2136
+ # Catches: المديره→المدير, للطالبه→للطالب
2137
+ if orig_text and corr_text:
2138
+ _o_g = orig_text.rstrip('.،؛؟!?!')
2139
+ _c_g = corr_text.rstrip('.،؛؟!?!')
2140
+ if (_o_g.endswith(('ه', 'ة')) and not _c_g.endswith(('ه', 'ة'))
2141
+ and (_c_g == _o_g[:-1] or len(_c_g) < len(_o_g))):
2142
+ logger.info(
2143
+ f"[GRAMMAR] Blocked feminine ending strip: "
2144
+ f"'{orig_text}'→'{corr_text}'"
2145
+ )
2146
+ continue
2147
+
2148
+
2149
+
2150
  # Evaluate grammar patterns early to bypass heuristic blocks.
2151
  _is_grammar_pattern = False
2152
  if orig_text and corr_text:
 
2208
  _is_grammar_pattern = True
2209
 
2210
 
2211
+ # ── FIX-42d: Grammar trailing letter addition guard ──
2212
+ # Block grammar changes that add ا/ي to end of IV words.
2213
+ # Catches: واجب→واجبا, معطف→معطفا
2214
+ # Must come AFTER _is_grammar_pattern so we don't block valid grammar.
2215
+ if not _is_grammar_pattern and orig_text and corr_text:
2216
+ _o_g2 = orig_text.rstrip('.،؛؟!?!')
2217
+ _c_g2 = corr_text.rstrip('.،؛؟!?!')
2218
+ if (len(_c_g2) == len(_o_g2) + 1 and _c_g2.startswith(_o_g2)
2219
+ and _c_g2[-1] in ('ا', 'ي')):
2220
+ logger.info(
2221
+ f"[GRAMMAR] Blocked trailing letter addition: "
2222
+ f"'{orig_text}'→'{corr_text}'"
2223
+ )
2224
+ continue
2225
+
2226
  # ── FIX-27a: Grammar structured data protection ──
2227
  # Block grammar diffs where the original contains digits.
2228
  # The grammar model corrupts dates/numbers/times/percentages.