Commit ·
08a99f9
1
Parent(s): 918deb4
fix: block grammar كان→كأن + punctuation mid-word split (الدفت.ر)
Browse files- src/app.py +30 -0
- src/nlp/punctuation/punctuation_rules.py +14 -0
src/app.py
CHANGED
|
@@ -1612,6 +1612,22 @@ def analyze_text():
|
|
| 1612 |
stage_label = 'grammar'
|
| 1613 |
if _is_spelling_only_change(orig_text, corr_text):
|
| 1614 |
stage_label = 'spelling'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1615 |
ctx.add_patch(
|
| 1616 |
stage_label, d['start'], d['end'],
|
| 1617 |
corr_text, confidence=1.0
|
|
@@ -1674,6 +1690,20 @@ def analyze_text():
|
|
| 1674 |
f"'{d.get('original','')}' \u2192 '{d.get('correction','')}' "
|
| 1675 |
f"(locked by {owner}[{ls}:{le}])"
|
| 1676 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1677 |
# Punctuation safety layer: reject non-punctuation changes
|
| 1678 |
if not validate_punctuation_diff(d):
|
| 1679 |
logger.info(
|
|
|
|
| 1612 |
stage_label = 'grammar'
|
| 1613 |
if _is_spelling_only_change(orig_text, corr_text):
|
| 1614 |
stage_label = 'spelling'
|
| 1615 |
+
|
| 1616 |
+
# ── Directional blocks for grammar (mirrors spelling filter) ──
|
| 1617 |
+
# Prevents grammar from making meaning-changing corrections
|
| 1618 |
+
# like كان→كأن ("was" → "as if").
|
| 1619 |
+
_GRAMMAR_BLOCKS = {
|
| 1620 |
+
'كان': {'كأن'}, 'كأن': {'كان'},
|
| 1621 |
+
'هذه': {'هذة'}, 'هذا': {'هذة', 'هذه'},
|
| 1622 |
+
'إلى': {'على', 'علي'}, 'على': {'إلى', 'علي'},
|
| 1623 |
+
'لكن': {'لاكن'}, 'ذلك': {'ذالك'},
|
| 1624 |
+
}
|
| 1625 |
+
if corr_text in _GRAMMAR_BLOCKS.get(orig_text, set()):
|
| 1626 |
+
logger.info(
|
| 1627 |
+
f"[GRAMMAR] Blocked directional: '{orig_text}'→'{corr_text}'"
|
| 1628 |
+
)
|
| 1629 |
+
continue
|
| 1630 |
+
|
| 1631 |
ctx.add_patch(
|
| 1632 |
stage_label, d['start'], d['end'],
|
| 1633 |
corr_text, confidence=1.0
|
|
|
|
| 1690 |
f"'{d.get('original','')}' \u2192 '{d.get('correction','')}' "
|
| 1691 |
f"(locked by {owner}[{ls}:{le}])"
|
| 1692 |
)
|
| 1693 |
+
# ── Mid-word split guard ──
|
| 1694 |
+
# Reject punctuation diffs where the original is NOT a complete
|
| 1695 |
+
# word — i.e., the character after the diff end is still Arabic.
|
| 1696 |
+
# This catches cases like الدفتر being split into الدفت.ر
|
| 1697 |
+
d_end = d['end']
|
| 1698 |
+
if d_end < len(ctx.current_text):
|
| 1699 |
+
next_ch = ctx.current_text[d_end]
|
| 1700 |
+
if '\u0600' <= next_ch <= '\u06FF':
|
| 1701 |
+
logger.info(
|
| 1702 |
+
f"[PUNC-SAFETY] Rejected mid-word split [{d['start']}:{d_end}] "
|
| 1703 |
+
f"'{d.get('original','')}' → '{d.get('correction','')}' "
|
| 1704 |
+
f"(next char '{next_ch}' is Arabic — word was split)"
|
| 1705 |
+
)
|
| 1706 |
+
continue
|
| 1707 |
# Punctuation safety layer: reject non-punctuation changes
|
| 1708 |
if not validate_punctuation_diff(d):
|
| 1709 |
logger.info(
|
src/nlp/punctuation/punctuation_rules.py
CHANGED
|
@@ -137,5 +137,19 @@ def validate_punctuation_diff(diff: dict) -> bool:
|
|
| 137 |
if punct_delta > MAX_PUNCT_DELTA:
|
| 138 |
return False
|
| 139 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
return True
|
| 141 |
|
|
|
|
| 137 |
if punct_delta > MAX_PUNCT_DELTA:
|
| 138 |
return False
|
| 139 |
|
| 140 |
+
# ── Rule 6: Reject mid-word punctuation insertion ──
|
| 141 |
+
# If the correction ends with a punctuation mark followed by nothing,
|
| 142 |
+
# but the original word is a PREFIX of a longer word in context,
|
| 143 |
+
# this indicates mid-word split (e.g. الدفت→الدفت. when word was الدفتر).
|
| 144 |
+
# Detect by checking if correction has punctuation NOT at word boundary.
|
| 145 |
+
for pc in ARABIC_PUNCT_CHARS:
|
| 146 |
+
if pc in correction:
|
| 147 |
+
# Check if punctuation is followed by an Arabic letter (mid-word)
|
| 148 |
+
idx = correction.find(pc)
|
| 149 |
+
if idx >= 0 and idx < len(correction) - 1:
|
| 150 |
+
next_char = correction[idx + 1]
|
| 151 |
+
if '\u0600' <= next_char <= '\u06FF':
|
| 152 |
+
return False
|
| 153 |
+
|
| 154 |
return True
|
| 155 |
|