youssefreda9 commited on
Commit
c29d4de
·
1 Parent(s): 078df28

fix(nlp): remove overly restrictive rules 0b and 0c from punctuation safety guards that blocked valid commas and colons

Browse files
src/nlp/punctuation/punctuation_rules.py CHANGED
@@ -198,47 +198,7 @@ def validate_punctuation_diff(diff: dict, full_text: str = '') -> bool:
198
  )
199
  return False
200
 
201
- # ── Rule 0b (Batch 4): Reject punct insertion when original has no punctuation ──
202
- # If the original text has zero Arabic punctuation and the correction
203
- # only adds commas/semicolons (not at the very end), it's overcorrection.
204
- # This catches "already correct" texts that PuncAra sprinkles with commas.
205
- orig_punct_count_r0b = sum(1 for c in original if c in ARABIC_PUNCT_CHARS)
206
- if orig_punct_count_r0b == 0:
207
- corr_punct_count_r0b = sum(1 for c in correction if c in ARABIC_PUNCT_CHARS)
208
- if corr_punct_count_r0b > 0:
209
- # Only allow if adding a single period/question at the very end
210
- stripped_corr = correction.rstrip()
211
- if stripped_corr and stripped_corr[-1] in '.؟?!':
212
- # This is terminal punct (already handled by Rule 0)
213
- pass
214
- else:
215
- # Mid-sentence punct insertion on a clean sentence → reject
216
- logger.info(
217
- f"[PUNC-SAFETY] Rejected mid-sentence punct insertion on clean text: "
218
- f"'{original}' → '{correction}'"
219
- )
220
- return False
221
-
222
- # ── Rule 0c (Batch 4 + FIX-26): Reject punctuation rearrangement/substitution ──
223
- # When original already has punctuation and the correction merely MOVES,
224
- # SUBSTITUTES, or STACKS marks (e.g., ، → : or ، → ؛ or ؟ → ؟!), reject.
225
- # The PuncAra model should NOT replace or pile onto existing punctuation —
226
- # a sentence that already ends with punctuation must never get a second
227
- # mark added next to it.
228
- orig_punct_count_r0c = sum(1 for c in original if c in ARABIC_PUNCT_CHARS)
229
- corr_punct_count_r0c = sum(1 for c in correction if c in ARABIC_PUNCT_CHARS)
230
- if orig_punct_count_r0c > 0 and corr_punct_count_r0c > 0:
231
- # Both have punctuation — check if alpha content is the same
232
- orig_alpha_r0c = re.sub(r'[.,،؛؟!:;?\s]', '', original)
233
- corr_alpha_r0c = re.sub(r'[.,،؛؟!:;?\s]', '', correction)
234
- if _normalize_for_comparison(orig_alpha_r0c) == _normalize_for_comparison(corr_alpha_r0c):
235
- # Same word content, but punct changed — reject any punct modification,
236
- # whether it's a substitution or an addition on top of existing punct.
237
- logger.info(
238
- f"[PUNC-SAFETY] Rejected punct substitution/stacking: "
239
- f"'{original}' → '{correction}'"
240
- )
241
- return False
242
 
243
  # ── Rule 1: Alphabetic content must be identical after normalization ──
244
  orig_alpha = re.sub(r'[.,،؛؟!:;?\s]', '', original)
 
198
  )
199
  return False
200
 
201
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
202
 
203
  # ── Rule 1: Alphabetic content must be identical after normalization ──
204
  orig_alpha = re.sub(r'[.,،؛؟!:;?\s]', '', original)