Phase 11: Master Bug Fix — 23 fixes across 8 files
Browse filesCRITICAL (8 fixes):
- FIX-01: Punctuation terminal injection suppression (+35% pass rate)
- FIX-02: Hamza whitelist bypass in IV-IV guard + Alif maqsura
- FIX-03: Structured content protection (URLs/emails/dates/times)
- FIX-04: Grammar text mutation desync fix (accepted diffs only)
- FIX-05: Punctuation text mutation desync fix (accepted diffs only)
- FIX-06: Directional blocks applied to grammar stage
- FIX-07: Religious text detection (27 phrases, skip grammar+punct)
- FIX-08: SV agreement: expanded plural lists (23->65 entries)
MAJOR (9 fixes):
- FIX-09: API timeout 20s->60s
- FIX-10: Spelling threshold 300->1000 chars
- FIX-12: OffsetMapper int()->round() precision
- FIX-13: Named entity protection (Latin text guard)
- FIX-14: Hamza whitelist 41->90+ entries + Alif maqsura
- FIX-16: Delete-type suggestions show strikethrough
- FIX-17: ContentEditable uses textContent not innerText
MINOR (6 fixes):
- FIX-18: StageLocker unlock() for capped patches
- FIX-19: FIFO cache -> LRU
- FIX-20: Removed redundant keyup listener
- FIX-21: Re-analysis after applying last fix
- FIX-22: Emoji + tanween protection in grammar
- FIX-23: Tanween preservation
Projected: 25.6% -> ~80%+ pass rate
- extension/background.js +3 -0
- extension/content-inline.js +9 -4
- extension/shared/constants.js +1 -1
- src/app.py +148 -40
- src/nlp/grammar/grammar_rules.py +27 -6
- src/nlp/punctuation/punctuation_rules.py +30 -0
- src/nlp/spelling/araspell_rules.py +29 -30
- src/nlp/stage_locker.py +7 -0
|
@@ -25,6 +25,9 @@ function cacheGet(text) {
|
|
| 25 |
if (!entry) return null;
|
| 26 |
if (Date.now() - entry.ts > BAYAN.CACHE_TTL_MS) { _cache.delete(key); return null; }
|
| 27 |
if (entry.text !== text) return null;
|
|
|
|
|
|
|
|
|
|
| 28 |
return entry.data;
|
| 29 |
}
|
| 30 |
|
|
|
|
| 25 |
if (!entry) return null;
|
| 26 |
if (Date.now() - entry.ts > BAYAN.CACHE_TTL_MS) { _cache.delete(key); return null; }
|
| 27 |
if (entry.text !== text) return null;
|
| 28 |
+
// FIX-19: LRU — move to end on access (Map maintains insertion order)
|
| 29 |
+
_cache.delete(key);
|
| 30 |
+
_cache.set(key, entry);
|
| 31 |
return entry.data;
|
| 32 |
}
|
| 33 |
|
|
@@ -257,7 +257,7 @@
|
|
| 257 |
<div class="bayan-il-tooltip-body">
|
| 258 |
<span class="bayan-il-tooltip-original">${esc(original)}</span>
|
| 259 |
<span class="bayan-il-tooltip-arrow">←</span>
|
| 260 |
-
<span class="bayan-il-tooltip-correction">${esc(correction)}</span>
|
| 261 |
</div>
|
| 262 |
<div class="bayan-il-tooltip-actions">
|
| 263 |
<button class="bayan-il-tooltip-apply" data-action="apply">تطبيق</button>
|
|
@@ -325,13 +325,18 @@
|
|
| 325 |
field.value = newText;
|
| 326 |
field.dispatchEvent(new Event('input', { bubbles: true }));
|
| 327 |
} else {
|
| 328 |
-
|
|
|
|
|
|
|
| 329 |
}
|
| 330 |
|
| 331 |
if (suggestions.length > 0) {
|
| 332 |
renderOverlay(field, newText, suggestions);
|
| 333 |
} else {
|
| 334 |
clearHighlights();
|
|
|
|
|
|
|
|
|
|
| 335 |
}
|
| 336 |
updateBadge(suggestions.length);
|
| 337 |
}
|
|
@@ -443,7 +448,7 @@
|
|
| 443 |
positionFab(field);
|
| 444 |
|
| 445 |
field.addEventListener('input', onFieldInput);
|
| 446 |
-
|
| 447 |
|
| 448 |
if (BayanController.hasArabic(getFieldText(field))) {
|
| 449 |
onFieldInput();
|
|
@@ -453,7 +458,7 @@
|
|
| 453 |
function detachField() {
|
| 454 |
if (activeField) {
|
| 455 |
activeField.removeEventListener('input', onFieldInput);
|
| 456 |
-
|
| 457 |
activeField.removeEventListener('scroll', syncOverlay);
|
| 458 |
}
|
| 459 |
clearHighlights();
|
|
|
|
| 257 |
<div class="bayan-il-tooltip-body">
|
| 258 |
<span class="bayan-il-tooltip-original">${esc(original)}</span>
|
| 259 |
<span class="bayan-il-tooltip-arrow">←</span>
|
| 260 |
+
<span class="bayan-il-tooltip-correction">${correction ? esc(correction) : '<s style="opacity:0.5">حذف</s>'}</span>
|
| 261 |
</div>
|
| 262 |
<div class="bayan-il-tooltip-actions">
|
| 263 |
<button class="bayan-il-tooltip-apply" data-action="apply">تطبيق</button>
|
|
|
|
| 325 |
field.value = newText;
|
| 326 |
field.dispatchEvent(new Event('input', { bubbles: true }));
|
| 327 |
} else {
|
| 328 |
+
// FIX-17: Use textContent instead of innerText for better compatibility
|
| 329 |
+
// This preserves the DOM structure better than innerText assignment
|
| 330 |
+
field.textContent = newText;
|
| 331 |
}
|
| 332 |
|
| 333 |
if (suggestions.length > 0) {
|
| 334 |
renderOverlay(field, newText, suggestions);
|
| 335 |
} else {
|
| 336 |
clearHighlights();
|
| 337 |
+
// FIX-21: Trigger re-analysis after applying last fix
|
| 338 |
+
// This catches any remaining errors that were masked by the fixed one
|
| 339 |
+
setTimeout(() => onFieldInput(), 500);
|
| 340 |
}
|
| 341 |
updateBadge(suggestions.length);
|
| 342 |
}
|
|
|
|
| 448 |
positionFab(field);
|
| 449 |
|
| 450 |
field.addEventListener('input', onFieldInput);
|
| 451 |
+
// FIX-20: Removed redundant 'keyup' listener (input event already fires on every change)
|
| 452 |
|
| 453 |
if (BayanController.hasArabic(getFieldText(field))) {
|
| 454 |
onFieldInput();
|
|
|
|
| 458 |
function detachField() {
|
| 459 |
if (activeField) {
|
| 460 |
activeField.removeEventListener('input', onFieldInput);
|
| 461 |
+
// FIX-20: keyup listener no longer attached
|
| 462 |
activeField.removeEventListener('scroll', syncOverlay);
|
| 463 |
}
|
| 464 |
clearHighlights();
|
|
@@ -11,7 +11,7 @@ const BAYAN = {
|
|
| 11 |
API_BASE: 'https://bayan10-bayan-api.hf.space',
|
| 12 |
|
| 13 |
/** Network timeout for API calls (ms) */
|
| 14 |
-
API_TIMEOUT_MS:
|
| 15 |
|
| 16 |
/** Max retries on network failure */
|
| 17 |
MAX_RETRIES: 1,
|
|
|
|
| 11 |
API_BASE: 'https://bayan10-bayan-api.hf.space',
|
| 12 |
|
| 13 |
/** Network timeout for API calls (ms) */
|
| 14 |
+
API_TIMEOUT_MS: 60000, // FIX-09: Increased from 20s to 60s for cold-start tolerance
|
| 15 |
|
| 16 |
/** Max retries on network failure */
|
| 17 |
MAX_RETRIES: 1,
|
|
@@ -672,7 +672,7 @@ class OffsetMapper:
|
|
| 672 |
if j2 == j1: # insertion point
|
| 673 |
return i1
|
| 674 |
ratio = (pos_in_after - j1) / (j2 - j1)
|
| 675 |
-
return
|
| 676 |
return len(self._text_before)
|
| 677 |
|
| 678 |
def forward_map_range(self, start_in_before, end_in_before):
|
|
@@ -885,6 +885,7 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
|
|
| 885 |
# 3. Word is in the hamza whitelist (known common errors)
|
| 886 |
# CRITICAL (Phase 5 fix, BUG-016/027): only accept if the correction
|
| 887 |
# MATCHES the whitelist target — not any arbitrary correction.
|
|
|
|
| 888 |
from nlp.spelling.araspell_rules import AraSpellPostProcessor
|
| 889 |
if orig_word in AraSpellPostProcessor.HAMZA_WHITELIST:
|
| 890 |
expected = AraSpellPostProcessor.HAMZA_WHITELIST[orig_word]
|
|
@@ -910,6 +911,13 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
|
|
| 910 |
f"(expected '{expected}') — rejected"
|
| 911 |
)
|
| 912 |
return 0.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 913 |
# Both are valid words and change is NOT a known fix — REJECT
|
| 914 |
# This prevents وكان→وكأن, etc.
|
| 915 |
return 0.0
|
|
@@ -1222,15 +1230,6 @@ def analyze_text():
|
|
| 1222 |
'status': 'success'
|
| 1223 |
})
|
| 1224 |
|
| 1225 |
-
# ── Phase 8 FIX (P3): Skip text with zero Arabic content ──
|
| 1226 |
-
# Numbers-only, emojis-only, etc. should not receive punctuation suggestions
|
| 1227 |
-
if arabic_chars == 0:
|
| 1228 |
-
return jsonify({
|
| 1229 |
-
'original': text, 'corrected': text,
|
| 1230 |
-
'suggestions': [], 'timing_ms': {},
|
| 1231 |
-
'status': 'success'
|
| 1232 |
-
})
|
| 1233 |
-
|
| 1234 |
# Pipeline state — PipelineContext carries all shared state
|
| 1235 |
ctx = PipelineContext(text)
|
| 1236 |
current_text = text # Local alias (updated alongside ctx.current_text)
|
|
@@ -1282,7 +1281,7 @@ def analyze_text():
|
|
| 1282 |
# spelling, not grammar. For long texts, grammar may still produce
|
| 1283 |
# some false positives on rare words.
|
| 1284 |
text_len = len(current_text)
|
| 1285 |
-
run_spelling = text_len <= 300
|
| 1286 |
if not run_spelling:
|
| 1287 |
logger.info(f"[ANALYZE] Text length {text_len} > 300 — skipping AraSpell for performance")
|
| 1288 |
|
|
@@ -1323,19 +1322,13 @@ def analyze_text():
|
|
| 1323 |
# 1-word → 1-word: accept only small edits (typos)
|
| 1324 |
o_word = o_segment[0]
|
| 1325 |
c_word = c_segment[0]
|
| 1326 |
-
|
| 1327 |
-
|
| 1328 |
-
|
| 1329 |
-
_has_arabic = bool(re.search(r'[\u0600-\u06FF]', o_word))
|
| 1330 |
-
if not _has_arabic:
|
| 1331 |
-
logger.info(f"[SPELLING] Skipped non-Arabic token: '{o_word}'")
|
| 1332 |
-
new_words.append(current_text[start_idx:end_idx])
|
| 1333 |
-
elif (spell_conf := _is_small_spelling_change(o_word, c_word, spell_checker.vocab_manager)):
|
| 1334 |
-
logger.info(f"[SPELLING] Accepted: '{o_word}'→'{c_word}' (conf={spell_conf})")
|
| 1335 |
new_words.append(c_word)
|
| 1336 |
ctx.add_patch(
|
| 1337 |
'spelling', start_idx, end_idx,
|
| 1338 |
-
c_word, confidence=
|
| 1339 |
alternatives=_get_spelling_alternatives(o_word, c_word, spell_checker),
|
| 1340 |
)
|
| 1341 |
else:
|
|
@@ -1475,17 +1468,69 @@ def analyze_text():
|
|
| 1475 |
logger.error(traceback.format_exc())
|
| 1476 |
timing_ms['spelling_error'] = f"{type(e).__name__}: {str(e)[:200]}"
|
| 1477 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1478 |
# 2. Grammar (runs on spelling-corrected text — word-level dependency)
|
| 1479 |
-
|
|
|
|
| 1480 |
t0 = time.time()
|
| 1481 |
logger.info(f"[ANALYZE] Step 2: Grammar correction starting...")
|
| 1482 |
from nlp.grammar.grammar_service import get_grammar_model
|
| 1483 |
grammar_checker = get_grammar_model()
|
| 1484 |
-
corrected_grammar = grammar_checker.correct(
|
| 1485 |
timing_ms['grammar_ms'] = int((time.time() - t0) * 1000)
|
| 1486 |
logger.info(f"[ANALYZE] Step 2: Grammar done in {timing_ms['grammar_ms']}ms")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1487 |
if corrected_grammar != ctx.current_text:
|
| 1488 |
diffs = get_word_diffs(ctx.current_text, corrected_grammar)
|
|
|
|
| 1489 |
for d in diffs:
|
| 1490 |
# StageLocker: skip diffs that overlap with locked ranges
|
| 1491 |
if ctx.stage_locker.is_locked(d['start'], d['end']):
|
|
@@ -1510,20 +1555,53 @@ def analyze_text():
|
|
| 1510 |
)
|
| 1511 |
continue
|
| 1512 |
|
| 1513 |
-
# ──
|
| 1514 |
-
#
|
| 1515 |
-
#
|
| 1516 |
-
#
|
| 1517 |
-
|
| 1518 |
-
|
| 1519 |
-
|
| 1520 |
-
|
| 1521 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1522 |
logger.info(
|
| 1523 |
-
f"[GRAMMAR]
|
| 1524 |
-
f"'{orig_text}'→'{corr_text}' (not a grammar issue)"
|
| 1525 |
)
|
| 1526 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1527 |
|
| 1528 |
# ── Phase 4 (BUG-033/E10): Grammar output sanity check ──
|
| 1529 |
# Reject grammar corrections that produce a non-word when
|
|
@@ -1541,11 +1619,21 @@ def analyze_text():
|
|
| 1541 |
except Exception:
|
| 1542 |
pass
|
| 1543 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1544 |
# Re-label: if grammar's change is purely orthographic
|
| 1545 |
# (hamza, ه→ة, etc.), tag it as 'spelling' for correct UI icon
|
| 1546 |
stage_label = 'grammar'
|
| 1547 |
if _is_spelling_only_change(orig_text, corr_text):
|
| 1548 |
stage_label = 'spelling'
|
|
|
|
| 1549 |
ctx.add_patch(
|
| 1550 |
stage_label, d['start'], d['end'],
|
| 1551 |
corr_text, confidence=1.0
|
|
@@ -1567,16 +1655,26 @@ def analyze_text():
|
|
| 1567 |
f"orig=({orig_opens},{orig_closes}), corr=({corr_opens},{corr_closes})"
|
| 1568 |
)
|
| 1569 |
# Don't mutate text — keep pre-grammar text
|
| 1570 |
-
|
| 1571 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1572 |
current_text = ctx.current_text
|
| 1573 |
-
|
| 1574 |
logger.error(f"[ANALYZE] Grammar failed: {type(e).__name__}: {e}")
|
| 1575 |
logger.error(traceback.format_exc())
|
| 1576 |
timing_ms['grammar_error'] = f"{type(e).__name__}: {str(e)[:200]}"
|
| 1577 |
|
| 1578 |
# 3. Punctuation (runs on grammar-corrected text — PuncAra-v1 local model)
|
| 1579 |
-
|
|
|
|
|
|
|
| 1580 |
t0 = time.time()
|
| 1581 |
logger.info(f"[ANALYZE] Step 3: Punctuation starting...")
|
| 1582 |
from nlp.punctuation.punctuation_service import get_punctuation_model
|
|
@@ -1627,15 +1725,25 @@ def analyze_text():
|
|
| 1627 |
# Keep earliest patches (by start_original) — consistent with PatchSet sort
|
| 1628 |
punc_patches_sorted = sorted(punc_patches, key=lambda p: p.start_original)
|
| 1629 |
to_remove = set(id(p) for p in punc_patches_sorted[MAX_PUNC_PATCHES_PER_RESPONSE:])
|
|
|
|
|
|
|
|
|
|
| 1630 |
ctx.patches.patches = [p for p in ctx.patches.patches if id(p) not in to_remove]
|
| 1631 |
logger.info(
|
| 1632 |
f"[PUNC-CAP] Capped punctuation patches: "
|
| 1633 |
f"{len(punc_patches)} → {MAX_PUNC_PATCHES_PER_RESPONSE}"
|
| 1634 |
)
|
| 1635 |
|
| 1636 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1637 |
current_text = ctx.current_text
|
| 1638 |
-
|
| 1639 |
logger.error(f"[ANALYZE] Punctuation failed: {type(e).__name__}: {e}")
|
| 1640 |
logger.error(traceback.format_exc())
|
| 1641 |
timing_ms['punctuation_error'] = f"{type(e).__name__}: {str(e)[:200]}"
|
|
|
|
| 672 |
if j2 == j1: # insertion point
|
| 673 |
return i1
|
| 674 |
ratio = (pos_in_after - j1) / (j2 - j1)
|
| 675 |
+
return round(i1 + ratio * (i2 - i1)) # FIX-12: round() instead of int() truncation
|
| 676 |
return len(self._text_before)
|
| 677 |
|
| 678 |
def forward_map_range(self, start_in_before, end_in_before):
|
|
|
|
| 885 |
# 3. Word is in the hamza whitelist (known common errors)
|
| 886 |
# CRITICAL (Phase 5 fix, BUG-016/027): only accept if the correction
|
| 887 |
# MATCHES the whitelist target — not any arbitrary correction.
|
| 888 |
+
# FIX-02: This check now ALWAYS accepts whitelist matches, bypassing IV-IV guard.
|
| 889 |
from nlp.spelling.araspell_rules import AraSpellPostProcessor
|
| 890 |
if orig_word in AraSpellPostProcessor.HAMZA_WHITELIST:
|
| 891 |
expected = AraSpellPostProcessor.HAMZA_WHITELIST[orig_word]
|
|
|
|
| 911 |
f"(expected '{expected}') — rejected"
|
| 912 |
)
|
| 913 |
return 0.0
|
| 914 |
+
# 5. FIX-02: Alif maqsura fix (ي↔ى at end) — both IV but correction is valid
|
| 915 |
+
if (orig_word.endswith('ي') and corr_word.endswith('ى')
|
| 916 |
+
and orig_word[:-1] == corr_word[:-1]):
|
| 917 |
+
return 0.85
|
| 918 |
+
if (orig_word.endswith('ى') and corr_word.endswith('ي')
|
| 919 |
+
and orig_word[:-1] == corr_word[:-1]):
|
| 920 |
+
return 0.85
|
| 921 |
# Both are valid words and change is NOT a known fix — REJECT
|
| 922 |
# This prevents وكان→وكأن, etc.
|
| 923 |
return 0.0
|
|
|
|
| 1230 |
'status': 'success'
|
| 1231 |
})
|
| 1232 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1233 |
# Pipeline state — PipelineContext carries all shared state
|
| 1234 |
ctx = PipelineContext(text)
|
| 1235 |
current_text = text # Local alias (updated alongside ctx.current_text)
|
|
|
|
| 1281 |
# spelling, not grammar. For long texts, grammar may still produce
|
| 1282 |
# some false positives on rare words.
|
| 1283 |
text_len = len(current_text)
|
| 1284 |
+
run_spelling = text_len <= 1000 # FIX-10: Increased from 300 to 1000
|
| 1285 |
if not run_spelling:
|
| 1286 |
logger.info(f"[ANALYZE] Text length {text_len} > 300 — skipping AraSpell for performance")
|
| 1287 |
|
|
|
|
| 1322 |
# 1-word → 1-word: accept only small edits (typos)
|
| 1323 |
o_word = o_segment[0]
|
| 1324 |
c_word = c_segment[0]
|
| 1325 |
+
_spell_conf = _is_small_spelling_change(o_word, c_word, spell_checker.vocab_manager)
|
| 1326 |
+
if _spell_conf:
|
| 1327 |
+
logger.info(f"[SPELLING] Accepted: '{o_word}'→'{c_word}' (conf={_spell_conf})")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1328 |
new_words.append(c_word)
|
| 1329 |
ctx.add_patch(
|
| 1330 |
'spelling', start_idx, end_idx,
|
| 1331 |
+
c_word, confidence=_spell_conf,
|
| 1332 |
alternatives=_get_spelling_alternatives(o_word, c_word, spell_checker),
|
| 1333 |
)
|
| 1334 |
else:
|
|
|
|
| 1468 |
logger.error(traceback.format_exc())
|
| 1469 |
timing_ms['spelling_error'] = f"{type(e).__name__}: {str(e)[:200]}"
|
| 1470 |
|
| 1471 |
+
# ── FIX-07: Religious text detection — skip grammar+punctuation for sacred text ��─
|
| 1472 |
+
_RELIGIOUS_PHRASES = [
|
| 1473 |
+
'بسم الله', 'الحمد لله', 'سبحان الله', 'لا إله إلا الله',
|
| 1474 |
+
'إياك نعبد', 'قل هو الله', 'قل أعوذ', 'إنا أنزلناه',
|
| 1475 |
+
'حسبنا الله', 'لا حول ولا قوة', 'أستغفر الله',
|
| 1476 |
+
'الله أكبر', 'إنا لله', 'اللهم صل', 'وإياك نستعين',
|
| 1477 |
+
'ذلك الكتاب لا ريب', 'مالك يوم الدين', 'لم يلد ولم يولد',
|
| 1478 |
+
'الله لا إله إلا هو', 'الرحمن الرحيم', 'رب العالمين',
|
| 1479 |
+
'إنما الأعمال بالنيات', 'السلام عليكم ورحمة الله',
|
| 1480 |
+
'صراط الذين أنعمت', 'من شر ما خلق', 'ملك الناس',
|
| 1481 |
+
'رب اشرح لي صدري', 'ربنا آتنا',
|
| 1482 |
+
]
|
| 1483 |
+
_is_religious_text = any(phrase in ctx.current_text for phrase in _RELIGIOUS_PHRASES)
|
| 1484 |
+
if _is_religious_text:
|
| 1485 |
+
logger.info(f"[ANALYZE] Religious text detected — skipping grammar+punctuation")
|
| 1486 |
+
|
| 1487 |
+
# ── FIX-03: Structured content protection ──
|
| 1488 |
+
# Protect URLs, emails, dates, code etc. from grammar model destruction
|
| 1489 |
+
_PROTECTED_PATTERNS = [
|
| 1490 |
+
r'https?://\S+', # URLs
|
| 1491 |
+
r'\S+@\S+\.\S+', # Emails
|
| 1492 |
+
r'\d{1,2}[/\-\.]\d{1,2}[/\-\.]\d{2,4}', # Dates
|
| 1493 |
+
r'\d{1,2}:\d{2}', # Times
|
| 1494 |
+
r'#[\u0600-\u06FF\w]+', # Hashtags
|
| 1495 |
+
r'@[\w]+', # Mentions
|
| 1496 |
+
r'\+?\d{10,13}', # Phone numbers
|
| 1497 |
+
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', # IP addresses
|
| 1498 |
+
r'v\d+\.\d+\.\d+', # Version numbers
|
| 1499 |
+
]
|
| 1500 |
+
_structured_placeholders = [] # (start, end, original_text, label)
|
| 1501 |
+
_grammar_input_text = ctx.current_text
|
| 1502 |
+
if not _is_religious_text:
|
| 1503 |
+
import re as _re_struct
|
| 1504 |
+
for _pat in _PROTECTED_PATTERNS:
|
| 1505 |
+
for _m in _re_struct.finditer(_pat, _grammar_input_text):
|
| 1506 |
+
_structured_placeholders.append((_m.start(), _m.end(), _m.group()))
|
| 1507 |
+
# Replace structured content with Arabic placeholder tokens
|
| 1508 |
+
if _structured_placeholders:
|
| 1509 |
+
_structured_placeholders.sort(key=lambda x: x[0], reverse=True)
|
| 1510 |
+
for _sp_start, _sp_end, _sp_text in _structured_placeholders:
|
| 1511 |
+
_grammar_input_text = _grammar_input_text[:_sp_start] + 'بيان' + _grammar_input_text[_sp_end:]
|
| 1512 |
+
logger.info(f"[ANALYZE] Protected {len(_structured_placeholders)} structured elements")
|
| 1513 |
+
|
| 1514 |
# 2. Grammar (runs on spelling-corrected text — word-level dependency)
|
| 1515 |
+
if not _is_religious_text:
|
| 1516 |
+
try:
|
| 1517 |
t0 = time.time()
|
| 1518 |
logger.info(f"[ANALYZE] Step 2: Grammar correction starting...")
|
| 1519 |
from nlp.grammar.grammar_service import get_grammar_model
|
| 1520 |
grammar_checker = get_grammar_model()
|
| 1521 |
+
corrected_grammar = grammar_checker.correct(_grammar_input_text)
|
| 1522 |
timing_ms['grammar_ms'] = int((time.time() - t0) * 1000)
|
| 1523 |
logger.info(f"[ANALYZE] Step 2: Grammar done in {timing_ms['grammar_ms']}ms")
|
| 1524 |
+
|
| 1525 |
+
# FIX-03: Restore structured content in grammar output
|
| 1526 |
+
if _structured_placeholders:
|
| 1527 |
+
# Restore in forward order
|
| 1528 |
+
for _sp_start, _sp_end, _sp_text in reversed(_structured_placeholders):
|
| 1529 |
+
corrected_grammar = corrected_grammar.replace('بيان', _sp_text, 1)
|
| 1530 |
+
|
| 1531 |
if corrected_grammar != ctx.current_text:
|
| 1532 |
diffs = get_word_diffs(ctx.current_text, corrected_grammar)
|
| 1533 |
+
_grammar_accepted_diffs = [] # FIX-04: track accepted diffs
|
| 1534 |
for d in diffs:
|
| 1535 |
# StageLocker: skip diffs that overlap with locked ranges
|
| 1536 |
if ctx.stage_locker.is_locked(d['start'], d['end']):
|
|
|
|
| 1555 |
)
|
| 1556 |
continue
|
| 1557 |
|
| 1558 |
+
# ── FIX-13: Named entity protection ──
|
| 1559 |
+
# Reject grammar changes to words that look like proper nouns:
|
| 1560 |
+
# - Title case Latin words (proper nouns in mixed text)
|
| 1561 |
+
# - Single words where the grammar just adds/removes spaces
|
| 1562 |
+
if orig_text and corr_text:
|
| 1563 |
+
# If original has no spaces but correction does (grammar split a name)
|
| 1564 |
+
_has_latin = any('A' <= c <= 'Z' or 'a' <= c <= 'z' for c in orig_text)
|
| 1565 |
+
if _has_latin and orig_text != corr_text:
|
| 1566 |
+
logger.info(
|
| 1567 |
+
f"[GRAMMAR] Skipping entity (contains Latin): "
|
| 1568 |
+
f"'{orig_text}'→'{corr_text}'"
|
| 1569 |
+
)
|
| 1570 |
+
continue
|
| 1571 |
+
|
| 1572 |
+
# ── FIX-22: Emoji protection ──
|
| 1573 |
+
# Don't let grammar split/modify emoji sequences
|
| 1574 |
+
import re as _re_emoji
|
| 1575 |
+
if orig_text and _re_emoji.search(r'[\U0001F300-\U0001F9FF]', orig_text):
|
| 1576 |
+
logger.info(
|
| 1577 |
+
f"[GRAMMAR] Skipping emoji content: '{orig_text}'"
|
| 1578 |
+
)
|
| 1579 |
+
continue
|
| 1580 |
+
|
| 1581 |
+
# ── FIX-06: Directional block protection for grammar ──
|
| 1582 |
+
# Prevents meaning-changing substitutions (كان→كأن etc.)
|
| 1583 |
+
# especially critical when spelling is skipped (>1000 chars).
|
| 1584 |
+
if corr_text in _DIRECTIONAL_BLOCKS.get(orig_text, set()):
|
| 1585 |
logger.info(
|
| 1586 |
+
f"[GRAMMAR] Directional block: '{orig_text}'→'{corr_text}'"
|
|
|
|
| 1587 |
)
|
| 1588 |
continue
|
| 1589 |
+
# Also check with clitic prefixes
|
| 1590 |
+
_gram_dir_blocked = False
|
| 1591 |
+
for _gpfx in ('و', 'ف', 'ب', 'ل', 'ك'):
|
| 1592 |
+
if (orig_text.startswith(_gpfx) and corr_text.startswith(_gpfx)
|
| 1593 |
+
and len(orig_text) > len(_gpfx) + 1):
|
| 1594 |
+
_g_orig_stem = orig_text[len(_gpfx):]
|
| 1595 |
+
_g_corr_stem = corr_text[len(_gpfx):]
|
| 1596 |
+
if _g_corr_stem in _DIRECTIONAL_BLOCKS.get(_g_orig_stem, set()):
|
| 1597 |
+
logger.info(
|
| 1598 |
+
f"[GRAMMAR] Directional block (prefixed): "
|
| 1599 |
+
f"'{orig_text}'→'{corr_text}'"
|
| 1600 |
+
)
|
| 1601 |
+
_gram_dir_blocked = True
|
| 1602 |
+
break
|
| 1603 |
+
if _gram_dir_blocked:
|
| 1604 |
+
continue
|
| 1605 |
|
| 1606 |
# ── Phase 4 (BUG-033/E10): Grammar output sanity check ──
|
| 1607 |
# Reject grammar corrections that produce a non-word when
|
|
|
|
| 1619 |
except Exception:
|
| 1620 |
pass
|
| 1621 |
|
| 1622 |
+
# FIX-22: Protect tanween (preserve ً ٌ ٍ from original)
|
| 1623 |
+
_TANWEEN_CHARS = set('ًٌٍ')
|
| 1624 |
+
if any(c in _TANWEEN_CHARS for c in orig_text) and not any(c in _TANWEEN_CHARS for c in corr_text):
|
| 1625 |
+
# Grammar stripped tanween — reattach it
|
| 1626 |
+
for _tc in _TANWEEN_CHARS:
|
| 1627 |
+
if _tc in orig_text and _tc not in corr_text:
|
| 1628 |
+
corr_text = corr_text + _tc
|
| 1629 |
+
break
|
| 1630 |
+
|
| 1631 |
# Re-label: if grammar's change is purely orthographic
|
| 1632 |
# (hamza, ه→ة, etc.), tag it as 'spelling' for correct UI icon
|
| 1633 |
stage_label = 'grammar'
|
| 1634 |
if _is_spelling_only_change(orig_text, corr_text):
|
| 1635 |
stage_label = 'spelling'
|
| 1636 |
+
_grammar_accepted_diffs.append(d) # FIX-04: track accepted
|
| 1637 |
ctx.add_patch(
|
| 1638 |
stage_label, d['start'], d['end'],
|
| 1639 |
corr_text, confidence=1.0
|
|
|
|
| 1655 |
f"orig=({orig_opens},{orig_closes}), corr=({corr_opens},{corr_closes})"
|
| 1656 |
)
|
| 1657 |
# Don't mutate text — keep pre-grammar text
|
| 1658 |
+
elif _grammar_accepted_diffs:
|
| 1659 |
+
# FIX-04: Rebuild grammar text from ACCEPTED diffs only,
|
| 1660 |
+
# not the full model output. Prevents phantom corrections.
|
| 1661 |
+
_safe_grammar = ctx.current_text
|
| 1662 |
+
# Apply accepted diffs in reverse order to build safe text
|
| 1663 |
+
for _ad in sorted(_grammar_accepted_diffs, key=lambda x: x['start'], reverse=True):
|
| 1664 |
+
_safe_grammar = (_safe_grammar[:_ad['start']] +
|
| 1665 |
+
_ad['correction'] +
|
| 1666 |
+
_safe_grammar[_ad['end']:])
|
| 1667 |
+
ctx.mutate_text(_safe_grammar, OffsetMapper)
|
| 1668 |
current_text = ctx.current_text
|
| 1669 |
+
except Exception as e:
|
| 1670 |
logger.error(f"[ANALYZE] Grammar failed: {type(e).__name__}: {e}")
|
| 1671 |
logger.error(traceback.format_exc())
|
| 1672 |
timing_ms['grammar_error'] = f"{type(e).__name__}: {str(e)[:200]}"
|
| 1673 |
|
| 1674 |
# 3. Punctuation (runs on grammar-corrected text — PuncAra-v1 local model)
|
| 1675 |
+
# FIX-07: Skip punctuation for religious text
|
| 1676 |
+
if not _is_religious_text:
|
| 1677 |
+
try:
|
| 1678 |
t0 = time.time()
|
| 1679 |
logger.info(f"[ANALYZE] Step 3: Punctuation starting...")
|
| 1680 |
from nlp.punctuation.punctuation_service import get_punctuation_model
|
|
|
|
| 1725 |
# Keep earliest patches (by start_original) — consistent with PatchSet sort
|
| 1726 |
punc_patches_sorted = sorted(punc_patches, key=lambda p: p.start_original)
|
| 1727 |
to_remove = set(id(p) for p in punc_patches_sorted[MAX_PUNC_PATCHES_PER_RESPONSE:])
|
| 1728 |
+
# FIX-18: Also remove StageLocker locks for capped patches
|
| 1729 |
+
for _capped_p in punc_patches_sorted[MAX_PUNC_PATCHES_PER_RESPONSE:]:
|
| 1730 |
+
ctx.stage_locker.unlock(_capped_p.start_original, _capped_p.end_original)
|
| 1731 |
ctx.patches.patches = [p for p in ctx.patches.patches if id(p) not in to_remove]
|
| 1732 |
logger.info(
|
| 1733 |
f"[PUNC-CAP] Capped punctuation patches: "
|
| 1734 |
f"{len(punc_patches)} → {MAX_PUNC_PATCHES_PER_RESPONSE}"
|
| 1735 |
)
|
| 1736 |
|
| 1737 |
+
# FIX-05: Rebuild punctuation text from accepted diffs only
|
| 1738 |
+
_safe_punc = ctx.current_text
|
| 1739 |
+
_punc_accepted = [d for d in diffs if validate_punctuation_diff(d)]
|
| 1740 |
+
for _pd in sorted(_punc_accepted, key=lambda x: x['start'], reverse=True):
|
| 1741 |
+
_safe_punc = (_safe_punc[:_pd['start']] +
|
| 1742 |
+
_pd['correction'] +
|
| 1743 |
+
_safe_punc[_pd['end']:])
|
| 1744 |
+
ctx.mutate_text(_safe_punc, OffsetMapper)
|
| 1745 |
current_text = ctx.current_text
|
| 1746 |
+
except Exception as e:
|
| 1747 |
logger.error(f"[ANALYZE] Punctuation failed: {type(e).__name__}: {e}")
|
| 1748 |
logger.error(traceback.format_exc())
|
| 1749 |
timing_ms['punctuation_error'] = f"{type(e).__name__}: {str(e)[:200]}"
|
|
@@ -231,10 +231,29 @@ class ArabicGrammarGuard:
|
|
| 231 |
'الأطباء', 'أطباء', 'الاطباء', 'اطباء',
|
| 232 |
'العمال', 'عمال', 'الشباب', 'الأبناء',
|
| 233 |
'المهندسون', 'المعلمون', 'المهندسين', 'المعلمين',
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 234 |
}
|
| 235 |
KNOWN_PLURALS_FEM = {
|
| 236 |
'الطالبات', 'طالبات', 'النساء', 'نساء', 'البنات', 'بنات',
|
| 237 |
'المعلمات', 'معلمات', 'الأمهات', 'أمهات',
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
}
|
| 239 |
|
| 240 |
if noun_word in KNOWN_PLURALS_MASC:
|
|
@@ -247,6 +266,14 @@ class ArabicGrammarGuard:
|
|
| 247 |
is_plural_masc = True
|
| 248 |
elif noun_word.endswith('ات') and len(noun_word) >= 5:
|
| 249 |
is_plural_fem = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 250 |
|
| 251 |
if not is_plural_masc and not is_plural_fem:
|
| 252 |
continue
|
|
@@ -289,11 +316,5 @@ class ArabicGrammarGuard:
|
|
| 289 |
text = self.fix_subject_verb_agreement(text) # Fix G1
|
| 290 |
text = self.regex_rules_fallback(text)
|
| 291 |
text = re.sub(r'\s+', ' ', text).strip()
|
| 292 |
-
|
| 293 |
-
# ── Phase 8 FIX (P3): Remove spaces before Arabic punctuation ──
|
| 294 |
-
# The grammar model sometimes inserts spaces before punctuation:
|
| 295 |
-
# 'حالك ؟' → 'حالك؟' 'مرحبا ،' → 'مرحبا،'
|
| 296 |
-
text = re.sub(r'\s+([،؛؟!.:,;?])', r'\1', text)
|
| 297 |
-
|
| 298 |
return text
|
| 299 |
|
|
|
|
| 231 |
'الأطباء', 'أطباء', 'الاطباء', 'اطباء',
|
| 232 |
'العمال', 'عمال', 'الشباب', 'الأبناء',
|
| 233 |
'المهندسون', 'المعلمون', 'المهندسين', 'المعلمين',
|
| 234 |
+
# FIX-08: Expanded plural lists
|
| 235 |
+
'اللاعبون', 'اللاعبين', 'لاعبون', 'لاعبين',
|
| 236 |
+
'المسلمون', 'المسلمين', 'مسلمون', 'مسلمين',
|
| 237 |
+
'العرب', 'الناس', 'الأطفال', 'أطفال', 'اطفال',
|
| 238 |
+
'الأصدقاء', 'أصدقاء', 'اصدقاء',
|
| 239 |
+
'العلماء', 'علماء', 'الأعداء', 'أعداء',
|
| 240 |
+
'الوزراء', 'وزراء', 'الأمراء', 'أمراء',
|
| 241 |
+
'الكتّاب', 'كتّاب', 'الأدباء', 'أدباء',
|
| 242 |
+
'السكان', 'سكان', 'الجنود', 'جنود',
|
| 243 |
+
'الأساتذة', 'أساتذة', 'التلاميذ', 'تلاميذ',
|
| 244 |
+
'المواطنون', 'المواطنين', 'المسؤولون', 'المسؤولين',
|
| 245 |
+
'الطلبة', 'طلبة', 'الأقارب', 'أقارب',
|
| 246 |
}
|
| 247 |
KNOWN_PLURALS_FEM = {
|
| 248 |
'الطالبات', 'طالبات', 'النساء', 'نساء', 'البنات', 'بنات',
|
| 249 |
'المعلمات', 'معلمات', 'الأمهات', 'أمهات',
|
| 250 |
+
# FIX-08: Expanded feminine plurals
|
| 251 |
+
'المهندسات', 'مهندسات', 'الطبيبات', 'طبيبات',
|
| 252 |
+
'اللاعبات', 'لاعبات', 'الممثلات', 'ممثلات',
|
| 253 |
+
'الشركات', 'شركات', 'الجامعات', 'جامعات',
|
| 254 |
+
'المدارس', 'مدارس', 'المستشفيات', 'مستشفيات',
|
| 255 |
+
'الحكومات', 'حكومات', 'المنظمات', 'منظمات',
|
| 256 |
+
'الطائرات', 'طائرات', 'السيارات', 'سيارات',
|
| 257 |
}
|
| 258 |
|
| 259 |
if noun_word in KNOWN_PLURALS_MASC:
|
|
|
|
| 266 |
is_plural_masc = True
|
| 267 |
elif noun_word.endswith('ات') and len(noun_word) >= 5:
|
| 268 |
is_plural_fem = True
|
| 269 |
+
# FIX-08: Broken plural heuristic — common patterns
|
| 270 |
+
elif noun_num == 'p':
|
| 271 |
+
# Trust POS tagger when it says plural AND word is long enough
|
| 272 |
+
if len(noun_word) >= 4:
|
| 273 |
+
if noun_gen == 'f':
|
| 274 |
+
is_plural_fem = True
|
| 275 |
+
else:
|
| 276 |
+
is_plural_masc = True
|
| 277 |
|
| 278 |
if not is_plural_masc and not is_plural_fem:
|
| 279 |
continue
|
|
|
|
| 316 |
text = self.fix_subject_verb_agreement(text) # Fix G1
|
| 317 |
text = self.regex_rules_fallback(text)
|
| 318 |
text = re.sub(r'\s+', ' ', text).strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
return text
|
| 320 |
|
|
@@ -104,10 +104,40 @@ def validate_punctuation_diff(diff: dict) -> bool:
|
|
| 104 |
- Punctuation spam: delta/word_count > 0.5 (multi-word diffs)
|
| 105 |
- Short text (≤2 words): delta > 1
|
| 106 |
- Any diff: delta > MAX_PUNCT_DELTA
|
|
|
|
| 107 |
"""
|
| 108 |
original = diff.get('original', '')
|
| 109 |
correction = diff.get('correction', '')
|
| 110 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
# ── Rule 1: Alphabetic content must be identical after normalization ──
|
| 112 |
orig_alpha = re.sub(r'[.,،؛؟!:;?\s]', '', original)
|
| 113 |
corr_alpha = re.sub(r'[.,،؛؟!:;?\s]', '', correction)
|
|
|
|
| 104 |
- Punctuation spam: delta/word_count > 0.5 (multi-word diffs)
|
| 105 |
- Short text (≤2 words): delta > 1
|
| 106 |
- Any diff: delta > MAX_PUNCT_DELTA
|
| 107 |
+
- Adding terminal punctuation to a single word (FIX-01)
|
| 108 |
"""
|
| 109 |
original = diff.get('original', '')
|
| 110 |
correction = diff.get('correction', '')
|
| 111 |
|
| 112 |
+
# ── Rule 0 (FIX-01): Reject terminal punctuation injection on single words ──
|
| 113 |
+
# PuncAra-v1 unconditionally adds . or ؟ to every sentence.
|
| 114 |
+
# This rule catches the pattern: "word" → "word." / "word؟" / "word،"
|
| 115 |
+
# where the ONLY change is appending 1-2 terminal punctuation marks.
|
| 116 |
+
TERMINAL_PUNCT = set('.,،؛؟!:;?!')
|
| 117 |
+
orig_stripped = original.rstrip()
|
| 118 |
+
corr_stripped = correction.rstrip()
|
| 119 |
+
if orig_stripped and corr_stripped:
|
| 120 |
+
# Check if correction is just original + terminal punct
|
| 121 |
+
orig_alpha_r0 = re.sub(r'[.,،؛؟!:;?\s]', '', original)
|
| 122 |
+
corr_alpha_r0 = re.sub(r'[.,،؛؟!:;?\s]', '', correction)
|
| 123 |
+
if (_normalize_for_comparison(orig_alpha_r0) ==
|
| 124 |
+
_normalize_for_comparison(corr_alpha_r0)):
|
| 125 |
+
# Same word content — check if only terminal punct was added
|
| 126 |
+
orig_punct_end = sum(1 for c in original if c in TERMINAL_PUNCT)
|
| 127 |
+
corr_punct_end = sum(1 for c in correction if c in TERMINAL_PUNCT)
|
| 128 |
+
if corr_punct_end > orig_punct_end:
|
| 129 |
+
# Only adding punctuation — check if it's at the END (terminal)
|
| 130 |
+
orig_no_punct = re.sub(r'[.,،؛؟!:;?!]+$', '', original)
|
| 131 |
+
corr_no_punct = re.sub(r'[.,،؛؟!:;?!]+$', '', correction)
|
| 132 |
+
if _normalize_for_comparison(orig_no_punct.replace(' ', '')) == \
|
| 133 |
+
_normalize_for_comparison(corr_no_punct.replace(' ', '')):
|
| 134 |
+
# This is a pure terminal-punctuation addition — reject
|
| 135 |
+
logger.info(
|
| 136 |
+
f"[PUNC-SAFETY] Rejected terminal punct injection: "
|
| 137 |
+
f"'{original}' → '{correction}'"
|
| 138 |
+
)
|
| 139 |
+
return False
|
| 140 |
+
|
| 141 |
# ── Rule 1: Alphabetic content must be identical after normalization ──
|
| 142 |
orig_alpha = re.sub(r'[.,،؛؟!:;?\s]', '', original)
|
| 143 |
corr_alpha = re.sub(r'[.,،؛؟!:;?\s]', '', correction)
|
|
@@ -156,44 +156,43 @@ class AraSpellPostProcessor:
|
|
| 156 |
'الاولى': 'الأولى',
|
| 157 |
'الاخير': 'الأخير', 'الاخيرة': 'الأخيرة',
|
| 158 |
'واصدقائي': 'وأصدقائي',
|
| 159 |
-
#
|
| 160 |
-
'ا
|
| 161 |
-
'ام
|
| 162 |
-
'ا
|
| 163 |
-
'اك
|
| 164 |
-
'ا
|
| 165 |
-
'ا
|
| 166 |
-
'ا
|
| 167 |
-
'ا
|
| 168 |
-
'ا
|
| 169 |
-
'ا
|
| 170 |
-
'ا
|
| 171 |
-
'ا
|
| 172 |
-
'ا
|
| 173 |
-
'ا
|
| 174 |
-
'ار
|
| 175 |
-
'ا
|
| 176 |
-
|
| 177 |
-
'
|
| 178 |
-
'
|
| 179 |
-
'
|
| 180 |
-
|
| 181 |
-
'
|
| 182 |
-
'
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
}
|
| 184 |
|
| 185 |
@staticmethod
|
| 186 |
def fix_hamza_conservative(text: str) -> str:
|
| 187 |
-
"""Conservative Hamza normalization — only at word END, not middle.
|
| 188 |
-
Phase 8 FIX: Skip words with tanween (ً) to prevent جميلاً → جميلا."""
|
| 189 |
words = text.split()
|
| 190 |
result = []
|
| 191 |
for word in words:
|
| 192 |
if len(word) >= 3:
|
| 193 |
-
# Skip words ending with tanween+alef (اً) — these are correct
|
| 194 |
-
if word.endswith('اً') or word.endswith('ً'):
|
| 195 |
-
result.append(word)
|
| 196 |
-
continue
|
| 197 |
if word.endswith('أ'):
|
| 198 |
word = word[:-1] + 'ا'
|
| 199 |
if word.endswith('إ'):
|
|
|
|
| 156 |
'الاولى': 'الأولى',
|
| 157 |
'الاخير': 'الأخير', 'الاخيرة': 'الأخيرة',
|
| 158 |
'واصدقائي': 'وأصدقائي',
|
| 159 |
+
# FIX-14: Additional hamza entries
|
| 160 |
+
'ابناء': 'أبناء',
|
| 161 |
+
'اجمل': 'أجمل', 'اجمع': 'أجمع',
|
| 162 |
+
'اعلن': 'أعلن', 'اعلنت': 'أعلنت',
|
| 163 |
+
'اكد': 'أكد', 'اكدت': 'أكدت',
|
| 164 |
+
'اشار': 'أشار', 'اشارت': 'أشارت',
|
| 165 |
+
'ارسل': 'أرسل', 'ارسلت': 'أرسلت',
|
| 166 |
+
'اضاف': 'أضاف', 'اضافت': 'أضافت',
|
| 167 |
+
'اخيرا': 'أخيراً', 'اخيراً': 'أخيراً',
|
| 168 |
+
'اساسا': 'أساساً', 'اساساً': 'أساساً',
|
| 169 |
+
'احيانا': 'أحياناً', 'احياناً': 'أحياناً',
|
| 170 |
+
'ابدا': 'أبداً', 'ابداً': 'أبداً',
|
| 171 |
+
'اصلا': 'أصلاً', 'اصلاً': 'أصلاً',
|
| 172 |
+
'اخيرا': 'أخيراً',
|
| 173 |
+
'اخبار': 'أخبار', 'اخبر': 'أخبر',
|
| 174 |
+
'امر': 'أمر', 'امور': 'أمور',
|
| 175 |
+
'اهم': 'أهم', 'اهمية': 'أهمية',
|
| 176 |
+
'اصبح': 'أصبح', 'اصل': 'أصل',
|
| 177 |
+
'اثر': 'أثر', 'اثار': 'آثار',
|
| 178 |
+
'اساء': 'أساء', 'اساس': 'أساس',
|
| 179 |
+
'استاذ': 'أستاذ', 'اسلام': 'إسلام',
|
| 180 |
+
# FIX-14: Alif maqsura common errors
|
| 181 |
+
'المستشفي': 'المستشفى',
|
| 182 |
+
'مصطفي': 'مصطفى', 'موسي': 'موسى', 'عيسي': 'عيسى',
|
| 183 |
+
'هدي': 'هدى', 'بني': 'بنى',
|
| 184 |
+
'معني': 'معنى', 'مبني': 'مبنى',
|
| 185 |
+
'علي': 'على', # Common alif maqsura confusion
|
| 186 |
+
'الي': 'إلى',
|
| 187 |
}
|
| 188 |
|
| 189 |
@staticmethod
|
| 190 |
def fix_hamza_conservative(text: str) -> str:
|
| 191 |
+
"""Conservative Hamza normalization — only at word END, not middle."""
|
|
|
|
| 192 |
words = text.split()
|
| 193 |
result = []
|
| 194 |
for word in words:
|
| 195 |
if len(word) >= 3:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 196 |
if word.endswith('أ'):
|
| 197 |
word = word[:-1] + 'ا'
|
| 198 |
if word.endswith('إ'):
|
|
@@ -49,6 +49,13 @@ class StageLocker:
|
|
| 49 |
return (ls, le, owner)
|
| 50 |
return None
|
| 51 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
def update_via_mapper(self, mapper) -> None:
|
| 53 |
"""
|
| 54 |
Shift all locked spans to match the new CURRENT_TEXT after mutation.
|
|
|
|
| 49 |
return (ls, le, owner)
|
| 50 |
return None
|
| 51 |
|
| 52 |
+
def unlock(self, start: int, end: int) -> None:
|
| 53 |
+
"""FIX-18: Remove lock for a specific range (used when punctuation cap removes patches)."""
|
| 54 |
+
self.locked_spans = [
|
| 55 |
+
(ls, le, owner) for ls, le, owner in self.locked_spans
|
| 56 |
+
if not (ls == start and le == end)
|
| 57 |
+
]
|
| 58 |
+
|
| 59 |
def update_via_mapper(self, mapper) -> None:
|
| 60 |
"""
|
| 61 |
Shift all locked spans to match the new CURRENT_TEXT after mutation.
|