youssefreda9 commited on
Commit
08a99f9
·
1 Parent(s): 918deb4

fix: block grammar كان→كأن + punctuation mid-word split (الدفت.ر)

Browse files
src/app.py CHANGED
@@ -1612,6 +1612,22 @@ def analyze_text():
1612
  stage_label = 'grammar'
1613
  if _is_spelling_only_change(orig_text, corr_text):
1614
  stage_label = 'spelling'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1615
  ctx.add_patch(
1616
  stage_label, d['start'], d['end'],
1617
  corr_text, confidence=1.0
@@ -1674,6 +1690,20 @@ def analyze_text():
1674
  f"'{d.get('original','')}' \u2192 '{d.get('correction','')}' "
1675
  f"(locked by {owner}[{ls}:{le}])"
1676
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1677
  # Punctuation safety layer: reject non-punctuation changes
1678
  if not validate_punctuation_diff(d):
1679
  logger.info(
 
1612
  stage_label = 'grammar'
1613
  if _is_spelling_only_change(orig_text, corr_text):
1614
  stage_label = 'spelling'
1615
+
1616
+ # ── Directional blocks for grammar (mirrors spelling filter) ──
1617
+ # Prevents grammar from making meaning-changing corrections
1618
+ # like كان→كأن ("was" → "as if").
1619
+ _GRAMMAR_BLOCKS = {
1620
+ 'كان': {'كأن'}, 'كأن': {'كان'},
1621
+ 'هذه': {'هذة'}, 'هذا': {'هذة', 'هذه'},
1622
+ 'إلى': {'على', 'علي'}, 'على': {'إلى', 'علي'},
1623
+ 'لكن': {'لاكن'}, 'ذلك': {'ذالك'},
1624
+ }
1625
+ if corr_text in _GRAMMAR_BLOCKS.get(orig_text, set()):
1626
+ logger.info(
1627
+ f"[GRAMMAR] Blocked directional: '{orig_text}'→'{corr_text}'"
1628
+ )
1629
+ continue
1630
+
1631
  ctx.add_patch(
1632
  stage_label, d['start'], d['end'],
1633
  corr_text, confidence=1.0
 
1690
  f"'{d.get('original','')}' \u2192 '{d.get('correction','')}' "
1691
  f"(locked by {owner}[{ls}:{le}])"
1692
  )
1693
+ # ── Mid-word split guard ──
1694
+ # Reject punctuation diffs where the original is NOT a complete
1695
+ # word — i.e., the character after the diff end is still Arabic.
1696
+ # This catches cases like الدفتر being split into الدفت.ر
1697
+ d_end = d['end']
1698
+ if d_end < len(ctx.current_text):
1699
+ next_ch = ctx.current_text[d_end]
1700
+ if '\u0600' <= next_ch <= '\u06FF':
1701
+ logger.info(
1702
+ f"[PUNC-SAFETY] Rejected mid-word split [{d['start']}:{d_end}] "
1703
+ f"'{d.get('original','')}' → '{d.get('correction','')}' "
1704
+ f"(next char '{next_ch}' is Arabic — word was split)"
1705
+ )
1706
+ continue
1707
  # Punctuation safety layer: reject non-punctuation changes
1708
  if not validate_punctuation_diff(d):
1709
  logger.info(
src/nlp/punctuation/punctuation_rules.py CHANGED
@@ -137,5 +137,19 @@ def validate_punctuation_diff(diff: dict) -> bool:
137
  if punct_delta > MAX_PUNCT_DELTA:
138
  return False
139
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
140
  return True
141
 
 
137
  if punct_delta > MAX_PUNCT_DELTA:
138
  return False
139
 
140
+ # ── Rule 6: Reject mid-word punctuation insertion ──
141
+ # If the correction ends with a punctuation mark followed by nothing,
142
+ # but the original word is a PREFIX of a longer word in context,
143
+ # this indicates mid-word split (e.g. الدفت→الدفت. when word was الدفتر).
144
+ # Detect by checking if correction has punctuation NOT at word boundary.
145
+ for pc in ARABIC_PUNCT_CHARS:
146
+ if pc in correction:
147
+ # Check if punctuation is followed by an Arabic letter (mid-word)
148
+ idx = correction.find(pc)
149
+ if idx >= 0 and idx < len(correction) - 1:
150
+ next_char = correction[idx + 1]
151
+ if '\u0600' <= next_char <= '\u06FF':
152
+ return False
153
+
154
  return True
155