Fix: Reorder Jaccard filter in app.py to run after evaluating grammar bypass rules

src/app.py

@@ -2006,6 +2006,64 @@ def analyze_text():
                             )
                             continue
 
+                    # Evaluate grammar patterns early to bypass heuristic blocks.
+                    _is_grammar_pattern = False
+                    if orig_text and corr_text:
+                        # Case: ون/ان → ين (sound masculine plural case)
+                        if (orig_text.endswith('ون') and corr_text.endswith('ين') and
+                                orig_text[:-2] == corr_text[:-2]):
+                            _is_grammar_pattern = True
+                        # Nasb/Jazm: ون → وا (verb mood)
+                        elif (orig_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…').endswith('ون') and corr_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…').endswith('وا') and len(orig_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…')) >= 3):
+                            _o_cl = orig_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…')
+                            _c_cl = corr_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…')
+                            _o_stem = _o_cl[:-2]
+                            _c_stem = _c_cl[:-2]
+                            if _o_stem == _c_stem or (len(_o_stem) > 1 and _o_stem[1:] == _c_stem[1:] and _o_stem[0] in 'يت' and _c_stem[0] in 'يت'):
+                                _is_grammar_pattern = True
+                        # Five nouns: وك → اك/يك (أبوك→أباك, أخوك→أخيك)
+                        elif (len(orig_text) >= 3 and len(corr_text) >= 3 and
+                                orig_text[-2:] in ('وك', 'وه') and
+                                corr_text[-2:] in ('اك', 'يك', 'اه', 'يه')):
+                            _is_grammar_pattern = True
+                        # Dual: ان → ين (dual oblique)
+                        elif (orig_text.endswith('ان') and corr_text.endswith('ين') and
+                                orig_text[:-2] == corr_text[:-2] and len(orig_text) >= 4):
+                            _is_grammar_pattern = True
+                        # Demonstrative: هذان→هاتان, هاتان→هذان
+                        elif ({orig_text, corr_text} <= {'هذان', 'هاتان'}):
+                            _is_grammar_pattern = True
+                        # ── NEW: SV agreement suffix additions ──
+                        # Past tense masc plural: verb→verb+وا (ذهب→ذهبوا, حضر→حضروا)
+                        elif (corr_text.endswith('وا') and corr_text[:-2] == orig_text
+                                and len(orig_text) >= 3):
+                            _is_grammar_pattern = True
+                        # Past tense fem plural: verb→verb+ن (ذهب→ذهبن, حضر→حضرن)
+                        elif (corr_text.endswith('ن') and corr_text[:-1] == orig_text
+                                and len(orig_text) >= 3):
+                            _is_grammar_pattern = True
+                        # Present tense fem plural: ون → ن (يلعبون → يلعبن or يلعبون → تلعبن)
+                        elif (orig_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…').endswith('ون') and corr_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…').endswith('ن') and len(orig_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…')) >= 3):
+                            _o_cl = orig_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…')
+                            _c_cl = corr_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…')
+                            _o_stem = _o_cl[:-2]
+                            _c_stem = _c_cl[:-1]
+                            if _o_stem == _c_stem or (len(_o_stem) > 1 and _o_stem[1:] == _c_stem[1:] and _o_stem[0] in 'يت' and _c_stem[0] in 'يت'):
+                                _is_grammar_pattern = True
+                        # Present tense masc plural: يفعل→يفعلون (adding ون)
+                        elif (corr_text.endswith('ون') and corr_text[:-2] == orig_text
+                                and len(orig_text) >= 3):
+                            _is_grammar_pattern = True
+                        # Gender: adjective→adjective+ة (جميل→جميلة, كبير→كبيرة)
+                        elif (corr_text.endswith('ة') and corr_text[:-1] == orig_text
+                                and len(orig_text) >= 3):
+                            _is_grammar_pattern = True
+                        # Gender with ي: ذكي→ذكية
+                        elif (corr_text.endswith('ية') and corr_text[:-1] == orig_text[:-1] + 'ي'
+                                and orig_text.endswith('ي') and len(orig_text) >= 3):
+                            _is_grammar_pattern = True
+
+
                     # ── FIX-27a: Grammar structured data protection ──
                     # Block grammar diffs where the original contains digits.
                     # The grammar model corrupts dates/numbers/times/percentages.
@@ -2023,7 +2081,7 @@ def analyze_text():
                     # Block grammar diffs where the correction is too different
                     # from the original (character-level Jaccard < 0.5).
                     # Catches: القانون→القانين, يعزف→يعزفون, للإنسان→للإنسين
-                    if orig_text and corr_text and len(orig_text) > 2:
+                    if not _is_grammar_pattern and orig_text and corr_text and len(orig_text) > 2:
                         import re as _re_jac
                         # Strip punctuation/spaces for comparison
                         _o_chars = set(_re_jac.sub(r'[\s.,،؛؟!:;?]', '', orig_text))
@@ -2042,7 +2100,7 @@ def analyze_text():
                     # ── FIX-06: Directional block protection for grammar ──
                     # Prevents meaning-changing substitutions (كان→كأن etc.)
                     # especially critical when spelling is skipped (>1000 chars).
-                    if corr_text in _DIRECTIONAL_BLOCKS.get(orig_text, set()):
+                    if not _is_grammar_pattern and corr_text in _DIRECTIONAL_BLOCKS.get(orig_text, set()):
                         logger.info(
                             f"[GRAMMAR] Directional block: '{orig_text}'→'{corr_text}'"
                         )
@@ -2066,63 +2124,6 @@ def analyze_text():
                     if _gram_dir_blocked:
                         continue
 
-                    # ── Phase 4 (BUG-033/E10): Grammar output sanity check ──
-                    # Reject grammar corrections that produce a non-word when
-                    # the original was already a valid word. Mirrors spelling filter.
-                    # BUT: bypass for known grammar patterns (case endings, nasb, etc.)
-                    _is_grammar_pattern = False
-                    if orig_text and corr_text:
-                        # Case: ون/ان → ين (sound masculine plural case)
-                        if (orig_text.endswith('ون') and corr_text.endswith('ين') and
-                                orig_text[:-2] == corr_text[:-2]):
-                            _is_grammar_pattern = True
-                        # Nasb/Jazm: ون → وا (verb mood)
-                        elif (orig_text.endswith('ون') and corr_text.endswith('وا') and
-                                orig_text[:-2] == corr_text[:-2]):
-                            _is_grammar_pattern = True
-                        # Five nouns: وك → اك/يك (أبوك→أباك, أخوك→أخيك)
-                        elif (len(orig_text) >= 3 and len(corr_text) >= 3 and
-                                orig_text[-2:] in ('وك', 'وه') and
-                                corr_text[-2:] in ('اك', 'يك', 'اه', 'يه')):
-                            _is_grammar_pattern = True
-                        # Dual: ان → ين (dual oblique)
-                        elif (orig_text.endswith('ان') and corr_text.endswith('ين') and
-                                orig_text[:-2] == corr_text[:-2] and len(orig_text) >= 4):
-                            _is_grammar_pattern = True
-                        # Demonstrative: هذان→هاتان, هاتان→هذان
-                        elif ({orig_text, corr_text} <= {'هذان', 'هاتان'}):
-                            _is_grammar_pattern = True
-                        # ── NEW: SV agreement suffix additions ──
-                        # Past tense masc plural: verb→verb+وا (ذهب→ذهبوا, حضر→حضروا)
-                        elif (corr_text.endswith('وا') and corr_text[:-2] == orig_text
-                                and len(orig_text) >= 3):
-                            _is_grammar_pattern = True
-                        # Past tense fem plural: verb→verb+ن (ذهب→ذهبن, حضر→حضرن)
-                        elif (corr_text.endswith('ن') and corr_text[:-1] == orig_text
-                                and len(orig_text) >= 3):
-                            _is_grammar_pattern = True
-                        # Present tense fem plural: ون → ن (يلعبون → يلعبن or يلعبون → تلعبن)
-                        elif (orig_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…').endswith('ون') and corr_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…').endswith('ن') and len(orig_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…')) >= 3):
-                            _o_cl = orig_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…')
-                            _c_cl = corr_text.rstrip('.,،؛;:!؟?()[]{}«»"\'…')
-                            _o_stem = _o_cl[:-2]
-                            _c_stem = _c_cl[:-1]
-                            if _o_stem == _c_stem or (len(_o_stem) > 1 and _o_stem[1:] == _c_stem[1:] and _o_stem[0] in 'يت' and _c_stem[0] in 'يت'):
-                                _is_grammar_pattern = True
-                        # Present tense masc plural: يفعل→يفعلون (adding ون)
-                        elif (corr_text.endswith('ون') and corr_text[:-2] == orig_text
-                                and len(orig_text) >= 3):
-                            _is_grammar_pattern = True
-                        # Gender: adjective→adjective+ة (جميل→جميلة, كبير→كبيرة)
-                        elif (corr_text.endswith('ة') and corr_text[:-1] == orig_text
-                                and len(orig_text) >= 3):
-                            _is_grammar_pattern = True
-                        # Gender with ي: ذكي→ذكية
-                        elif (corr_text.endswith('ية') and corr_text[:-1] == orig_text[:-1] + 'ي'
-                                and orig_text.endswith('ي') and len(orig_text) >= 3):
-                            _is_grammar_pattern = True
-
-
                     if not _is_grammar_pattern:
                         if len(orig_text.split()) == 1 and len(corr_text.split()) == 1:
                             try: