youssefreda9 commited on
Commit
3f2c874
·
1 Parent(s): f5aa846

FIX-26: Reject punct substitution when original has punct

Browse files

Previously Rule 0c only rejected rearrangement (same punct chars in
different positions). Now also rejects substitution (comma→colon,
comma→semicolon) when the alphabetic content is unchanged.

Live tested: Fixes P013 (أحمد، كيف→أحمد كيف:) and H026
(الأمم، وبدونه→الأمم وبدونه؛). P012 and P015 already fixed.

src/nlp/punctuation/punctuation_rules.py CHANGED
@@ -159,20 +159,20 @@ def validate_punctuation_diff(diff: dict) -> bool:
159
  )
160
  return False
161
 
162
- # ── Rule 0c (Batch 4): Reject punctuation rearrangement ──
163
  # When original already has punctuation and the correction merely MOVES
164
- # marks to different positions (same count, same chars), reject.
165
- # e.g. "حالك؟ أنا" → "حالك أنا؟" = rearrangement
166
  orig_punct_count_r0c = sum(1 for c in original if c in ARABIC_PUNCT_CHARS)
167
  corr_punct_count_r0c = sum(1 for c in correction if c in ARABIC_PUNCT_CHARS)
168
  if orig_punct_count_r0c > 0 and corr_punct_count_r0c > 0:
169
- # Both have punctuation — check if it's just rearrangement
170
- orig_punct_set = sorted(c for c in original if c in ARABIC_PUNCT_CHARS)
171
- corr_punct_set = sorted(c for c in correction if c in ARABIC_PUNCT_CHARS)
172
- if orig_punct_set == corr_punct_set:
173
- # Same punct chars — only positions changed. This is rearrangement.
174
  logger.info(
175
- f"[PUNC-SAFETY] Rejected punct rearrangement: "
176
  f"'{original}' → '{correction}'"
177
  )
178
  return False
 
159
  )
160
  return False
161
 
162
+ # ── Rule 0c (Batch 4 + FIX-26): Reject punctuation rearrangement/substitution ──
163
  # When original already has punctuation and the correction merely MOVES
164
+ # or SUBSTITUTES marks (e.g., ، → : or ، → ؛), reject.
165
+ # The PuncAra model should NOT replace existing punctuation.
166
  orig_punct_count_r0c = sum(1 for c in original if c in ARABIC_PUNCT_CHARS)
167
  corr_punct_count_r0c = sum(1 for c in correction if c in ARABIC_PUNCT_CHARS)
168
  if orig_punct_count_r0c > 0 and corr_punct_count_r0c > 0:
169
+ # Both have punctuation — check if alpha content is the same
170
+ orig_alpha_r0c = re.sub(r'[.,،؛؟!:;?\s]', '', original)
171
+ corr_alpha_r0c = re.sub(r'[.,،؛؟!:;?\s]', '', correction)
172
+ if _normalize_for_comparison(orig_alpha_r0c) == _normalize_for_comparison(corr_alpha_r0c):
173
+ # Same word content, but punct changed — reject any punct modification
174
  logger.info(
175
+ f"[PUNC-SAFETY] Rejected punct substitution: "
176
  f"'{original}' → '{correction}'"
177
  )
178
  return False
tests/phase10/reports/phase10_results.json CHANGED
The diff for this file is too large to render. See raw diff