Commit ·
c29d4de
1
Parent(s): 078df28
fix(nlp): remove overly restrictive rules 0b and 0c from punctuation safety guards that blocked valid commas and colons
Browse files
src/nlp/punctuation/punctuation_rules.py
CHANGED
|
@@ -198,47 +198,7 @@ def validate_punctuation_diff(diff: dict, full_text: str = '') -> bool:
|
|
| 198 |
)
|
| 199 |
return False
|
| 200 |
|
| 201 |
-
|
| 202 |
-
# If the original text has zero Arabic punctuation and the correction
|
| 203 |
-
# only adds commas/semicolons (not at the very end), it's overcorrection.
|
| 204 |
-
# This catches "already correct" texts that PuncAra sprinkles with commas.
|
| 205 |
-
orig_punct_count_r0b = sum(1 for c in original if c in ARABIC_PUNCT_CHARS)
|
| 206 |
-
if orig_punct_count_r0b == 0:
|
| 207 |
-
corr_punct_count_r0b = sum(1 for c in correction if c in ARABIC_PUNCT_CHARS)
|
| 208 |
-
if corr_punct_count_r0b > 0:
|
| 209 |
-
# Only allow if adding a single period/question at the very end
|
| 210 |
-
stripped_corr = correction.rstrip()
|
| 211 |
-
if stripped_corr and stripped_corr[-1] in '.؟?!':
|
| 212 |
-
# This is terminal punct (already handled by Rule 0)
|
| 213 |
-
pass
|
| 214 |
-
else:
|
| 215 |
-
# Mid-sentence punct insertion on a clean sentence → reject
|
| 216 |
-
logger.info(
|
| 217 |
-
f"[PUNC-SAFETY] Rejected mid-sentence punct insertion on clean text: "
|
| 218 |
-
f"'{original}' → '{correction}'"
|
| 219 |
-
)
|
| 220 |
-
return False
|
| 221 |
-
|
| 222 |
-
# ── Rule 0c (Batch 4 + FIX-26): Reject punctuation rearrangement/substitution ──
|
| 223 |
-
# When original already has punctuation and the correction merely MOVES,
|
| 224 |
-
# SUBSTITUTES, or STACKS marks (e.g., ، → : or ، → ؛ or ؟ → ؟!), reject.
|
| 225 |
-
# The PuncAra model should NOT replace or pile onto existing punctuation —
|
| 226 |
-
# a sentence that already ends with punctuation must never get a second
|
| 227 |
-
# mark added next to it.
|
| 228 |
-
orig_punct_count_r0c = sum(1 for c in original if c in ARABIC_PUNCT_CHARS)
|
| 229 |
-
corr_punct_count_r0c = sum(1 for c in correction if c in ARABIC_PUNCT_CHARS)
|
| 230 |
-
if orig_punct_count_r0c > 0 and corr_punct_count_r0c > 0:
|
| 231 |
-
# Both have punctuation — check if alpha content is the same
|
| 232 |
-
orig_alpha_r0c = re.sub(r'[.,،؛؟!:;?\s]', '', original)
|
| 233 |
-
corr_alpha_r0c = re.sub(r'[.,،؛؟!:;?\s]', '', correction)
|
| 234 |
-
if _normalize_for_comparison(orig_alpha_r0c) == _normalize_for_comparison(corr_alpha_r0c):
|
| 235 |
-
# Same word content, but punct changed — reject any punct modification,
|
| 236 |
-
# whether it's a substitution or an addition on top of existing punct.
|
| 237 |
-
logger.info(
|
| 238 |
-
f"[PUNC-SAFETY] Rejected punct substitution/stacking: "
|
| 239 |
-
f"'{original}' → '{correction}'"
|
| 240 |
-
)
|
| 241 |
-
return False
|
| 242 |
|
| 243 |
# ── Rule 1: Alphabetic content must be identical after normalization ──
|
| 244 |
orig_alpha = re.sub(r'[.,،؛؟!:;?\s]', '', original)
|
|
|
|
| 198 |
)
|
| 199 |
return False
|
| 200 |
|
| 201 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 202 |
|
| 203 |
# ── Rule 1: Alphabetic content must be identical after normalization ──
|
| 204 |
orig_alpha = re.sub(r'[.,،؛؟!:;?\s]', '', original)
|