Commit ·
3f2c874
1
Parent(s): f5aa846
FIX-26: Reject punct substitution when original has punct
Browse filesPreviously Rule 0c only rejected rearrangement (same punct chars in
different positions). Now also rejects substitution (comma→colon,
comma→semicolon) when the alphabetic content is unchanged.
Live tested: Fixes P013 (أحمد، كيف→أحمد كيف:) and H026
(الأمم، وبدونه→الأمم وبدونه؛). P012 and P015 already fixed.
src/nlp/punctuation/punctuation_rules.py
CHANGED
|
@@ -159,20 +159,20 @@ def validate_punctuation_diff(diff: dict) -> bool:
|
|
| 159 |
)
|
| 160 |
return False
|
| 161 |
|
| 162 |
-
# ── Rule 0c (Batch 4): Reject punctuation rearrangement ──
|
| 163 |
# When original already has punctuation and the correction merely MOVES
|
| 164 |
-
# marks
|
| 165 |
-
#
|
| 166 |
orig_punct_count_r0c = sum(1 for c in original if c in ARABIC_PUNCT_CHARS)
|
| 167 |
corr_punct_count_r0c = sum(1 for c in correction if c in ARABIC_PUNCT_CHARS)
|
| 168 |
if orig_punct_count_r0c > 0 and corr_punct_count_r0c > 0:
|
| 169 |
-
# Both have punctuation — check if
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
if
|
| 173 |
-
# Same
|
| 174 |
logger.info(
|
| 175 |
-
f"[PUNC-SAFETY] Rejected punct
|
| 176 |
f"'{original}' → '{correction}'"
|
| 177 |
)
|
| 178 |
return False
|
|
|
|
| 159 |
)
|
| 160 |
return False
|
| 161 |
|
| 162 |
+
# ── Rule 0c (Batch 4 + FIX-26): Reject punctuation rearrangement/substitution ──
|
| 163 |
# When original already has punctuation and the correction merely MOVES
|
| 164 |
+
# or SUBSTITUTES marks (e.g., ، → : or ، → ؛), reject.
|
| 165 |
+
# The PuncAra model should NOT replace existing punctuation.
|
| 166 |
orig_punct_count_r0c = sum(1 for c in original if c in ARABIC_PUNCT_CHARS)
|
| 167 |
corr_punct_count_r0c = sum(1 for c in correction if c in ARABIC_PUNCT_CHARS)
|
| 168 |
if orig_punct_count_r0c > 0 and corr_punct_count_r0c > 0:
|
| 169 |
+
# Both have punctuation — check if alpha content is the same
|
| 170 |
+
orig_alpha_r0c = re.sub(r'[.,،؛؟!:;?\s]', '', original)
|
| 171 |
+
corr_alpha_r0c = re.sub(r'[.,،؛؟!:;?\s]', '', correction)
|
| 172 |
+
if _normalize_for_comparison(orig_alpha_r0c) == _normalize_for_comparison(corr_alpha_r0c):
|
| 173 |
+
# Same word content, but punct changed — reject any punct modification
|
| 174 |
logger.info(
|
| 175 |
+
f"[PUNC-SAFETY] Rejected punct substitution: "
|
| 176 |
f"'{original}' → '{correction}'"
|
| 177 |
)
|
| 178 |
return False
|
tests/phase10/reports/phase10_results.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|