feat: P1-P3 all 8 items - ta marbuta fix, UI feedback, caching, rate limiting, unit tests

hybrid_module.py

@@ -0,0 +1,147 @@
+
+# hybrid_module.py
+
+import torch
+import pickle
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+from huggingface_hub import hf_hub_download
+
+# ---------- Load Bigram ----------
+
+def load_bigram(repo_id="bayan10/AutoComplete", filename="bigram_model_v4.pkl"):
+    path = hf_hub_download(repo_id=repo_id, filename=filename)
+    with open(path, "rb") as f:
+        data = pickle.load(f)
+    return data["unigrams"], data["bigrams"]
+
+# ---------- Load GPT-2 ----------
+def load_gpt2(model_name="aubmindlab/aragpt2-base"):
+    tokenizer = GPT2Tokenizer.from_pretrained(model_name)
+    model = GPT2LMHeadModel.from_pretrained(model_name)
+    tokenizer.pad_token = tokenizer.eos_token
+    model.config.pad_token_id = tokenizer.eos_token_id
+    model.eval()
+    return tokenizer, model
+
+# ---------- GPT-2 scoring ----------
+def gpt2_next_token_probs(prefix, tokenizer, model, top_k=50):
+    inputs = tokenizer(
+        prefix,
+        return_tensors="pt",
+        truncation=True,
+        max_length=1024
+    )
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+        logits = outputs.logits[0, -1]
+
+    probs = torch.softmax(logits, dim=-1)
+    top_probs, top_ids = torch.topk(probs, top_k)
+
+    prob_dict = {}
+    for idx, prob in zip(top_ids, top_probs):
+        word = tokenizer.decode([idx]).strip()
+        if word:
+            prob_dict[word] = prob.item()
+
+    return prob_dict
+
+# ---------- Statistical autocomplete ----------
+
+
+def statistical_autocomplete(text, unigrams, bigrams, top_k=20):
+    tokens = text.strip().split()
+    if not tokens:
+        return []
+
+    last_word = tokens[-1]
+    candidates = []
+
+    if last_word in bigrams:
+        for w, c in bigrams[last_word].items():
+            if len(w) < 3 or w == last_word:
+                continue
+            candidates.append((w, c))
+
+    if not candidates:
+        for w, c in unigrams.items():
+            if len(w) < 3:
+                continue
+            candidates.append((w, c))
+
+    total = sum(c for _, c in candidates)
+    preds = [(w, c / total) for w, c in candidates]
+    preds.sort(key=lambda x: x[1], reverse=True)
+    preds = merge_similar_predictions(preds, top_k=top_k)
+    return preds[:top_k]
+
+# ---------- Hybrid autocomplete ----------
+def hybrid_autocomplete(prefix, unigrams, bigrams, tokenizer, model, alpha=0.6, k=5):
+    words = prefix.strip().split()
+    if len(words) < 1:
+        return []
+
+    last_word = words[-1]
+    if last_word not in bigrams:
+        return []
+
+    # -------- Statistical (Bigram) --------
+    stat_candidates = statistical_autocomplete(
+    prefix,
+    unigrams,
+    bigrams,
+    top_k=20
+)
+
+    # -------- Neural (GPT-2) — ONCE --------
+    gpt2_probs = gpt2_next_token_probs(prefix, tokenizer, model, top_k=50)
+
+    # -------- Hybrid scoring --------
+    results = []
+    for w, stat_p in stat_candidates:
+        neural_p = gpt2_probs.get(w, 1e-8)  # small value if not found
+        score = alpha * stat_p + (1 - alpha) * neural_p
+        results.append((w, score))
+
+    return sorted(results, key=lambda x: x[1], reverse=True)[:k]
+
+import re
+from collections import defaultdict
+
+def canonical_form(word):
+    word = re.sub("[إأآا]", "ا", word)
+    word = re.sub("ى", "ي", word)
+    word = re.sub("ؤ", "و", word)
+    word = re.sub("ئ", "ي", word)
+    word = re.sub("ة", "ه", word)
+    word = re.sub(r"[ًٌٍَُِّْ]", "", word)
+    return word
+
+
+
+def merge_similar_predictions(preds, top_k=20):
+    groups = defaultdict(lambda: {"score": 0.0, "words": []})
+
+    for w, p in preds:
+        key = canonical_form(w)
+        groups[key]["score"] += p
+        groups[key]["words"].append(w)
+
+    merged = sorted(
+        groups.values(),
+        key=lambda x: x["score"],
+        reverse=True
+    )
+
+    return [
+        (group["words"][0], group["score"])
+        for group in merged[:top_k]
+    ]
+
+
+
+
+
+
+

src/app.py

@@ -6,6 +6,9 @@ Provides API endpoints for the Bayan web application.
 import os
 import logging
 import time
+import hashlib
+from collections import OrderedDict
+from functools import wraps
 from flask import Flask, request, jsonify, Response
 from flask_cors import CORS
 from pathlib import Path
@@ -76,6 +79,119 @@ MAX_TEXT_LENGTH = 5000  # Maximum characters for input text
 MAX_SUMMARY_LENGTH = 512  # Maximum tokens for summary
 MIN_TEXT_LENGTH = 10  # Minimum characters for summarization
 
+# ── Response Cache (P3) ──
+# LRU cache for /api/analyze: hash(text) → (response_dict, timestamp)
+_ANALYZE_CACHE_MAX = 500
+_ANALYZE_CACHE_TTL = 300  # 5 minutes
+_analyze_cache = OrderedDict()
+
+# ── Rate Limiter (P3) ──
+_RATE_LIMIT_MAX = 30  # requests per window
+_RATE_LIMIT_WINDOW = 60  # seconds
+_rate_limit_store = {}  # ip → [(timestamp, ...)]
+
+# ── Ta Marbuta Dictionary (P2) ──
+# Common words where ه at the end should be ة
+_TA_MARBUTA_DICT = {
+    'المدرسه': 'المدرسة', 'الجامعه': 'الجامعة', 'المكتبه': 'المكتبة',
+    'الحياه': 'الحياة', 'الصلاه': 'الصلاة', 'الزكاه': 'الزكاة',
+    'القراءه': 'القراءة', 'الكتابه': 'الكتابة', 'المعرفه': 'المعرفة',
+    'الثقافه': 'الثقافة', 'السياسه': 'السياسة', 'الاقتصاديه': 'الاقتصادية',
+    'العربيه': 'العربية', 'الاسلاميه': 'الإسلامية', 'التربيه': 'التربية',
+    'الشريعه': 'الشريعة', 'الدوله': 'الدولة', 'الحكومه': 'الحكومة',
+    'المدينه': 'المدينة', 'القريه': 'القرية', 'الغرفه': 'الغرفة',
+    'السياره': 'السيارة', 'الطاوله': 'الطاولة', 'الرساله': 'الرسالة',
+    'المقاله': 'المقالة', 'الصحيفه': 'الصحيفة', 'الجريده': 'الجريدة',
+    'القصه': 'القصة', 'الروايه': 'الرواية', 'اللغه': 'اللغة',
+    'الفكره': 'الفكرة', 'الخطوه': 'الخطوة', 'المرحله': 'المرحلة',
+    'النتيجه': 'النتيجة', 'المشكله': 'المشكلة', 'الطريقه': 'الطريقة',
+    'الحاله': 'الحالة', 'الصوره': 'الصورة', 'القوه': 'القوة',
+    'الوحده': 'الوحدة', 'العلاقه': 'العلاقة', 'التجربه': 'التجربة',
+    'الحركه': 'الحركة', 'السلطه': 'السلطة', 'المنطقه': 'المنطقة',
+    'الساعه': 'الساعة', 'اللحظه': 'اللحظة', 'الفتره': 'الفترة',
+    'الاداره': 'الإدارة', 'البيئه': 'البيئة', 'الماده': 'المادة',
+    'الاسره': 'الأسرة', 'العائله': 'العائلة', 'الشركه': 'الشركة',
+    'المؤسسه': 'المؤسسة', 'المنظمه': 'المنظمة', 'الجمعيه': 'الجمعية',
+    'الوزاره': 'الوزارة', 'السفاره': 'السفارة', 'القياده': 'القيادة',
+    'الزياره': 'الزيارة', 'المحاوله': 'المحاولة', 'الدراسه': 'الدراسة',
+    'الممارسه': 'الممارسة', 'المتابعه': 'المتابعة', 'الخدمه': 'الخدمة',
+    'التقنيه': 'التقنية', 'الهندسه': 'الهندسة', 'الفلسفه': 'الفلسفة',
+    'مدرسه': 'مدرسة', 'جامعه': 'جامعة', 'مكتبه': 'مكتبة',
+    'حياه': 'حياة', 'صلاه': 'صلاة', 'زكاه': 'زكاة',
+    'لغه': 'لغة', 'قصه': 'قصة', 'فكره': 'فكرة',
+    'خطوه': 'خطوة', 'صوره': 'صورة', 'قوه': 'قوة',
+    'سياره': 'سيارة', 'رساله': 'رسالة', 'ساعه': 'ساعة',
+    'غرفه': 'غرفة', 'شركه': 'شركة', 'دوله': 'دولة',
+}
+
+
+def _fix_ta_marbuta(text):
+    """Fix common ه→ة errors at pipeline level using dictionary lookup."""
+    words = text.split()
+    fixed_words = []
+    changes = []
+    pos = 0
+    for word in words:
+        start = text.find(word, pos)
+        end = start + len(word)
+        # Check bare word
+        if word in _TA_MARBUTA_DICT:
+            fixed_words.append(_TA_MARBUTA_DICT[word])
+            changes.append({'start': start, 'end': end, 'original': word, 'correction': _TA_MARBUTA_DICT[word]})
+        # Check word ending in ه that should be ة (pattern match)
+        elif word.endswith('ه') and len(word) >= 3:
+            candidate = word[:-1] + 'ة'
+            if candidate in _TA_MARBUTA_DICT.values():
+                fixed_words.append(candidate)
+                changes.append({'start': start, 'end': end, 'original': word, 'correction': candidate})
+            else:
+                fixed_words.append(word)
+        else:
+            fixed_words.append(word)
+        pos = end
+    return ' '.join(fixed_words), changes
+
+
+def _check_rate_limit(ip):
+    """Check if IP has exceeded rate limit. Returns True if allowed."""
+    now = time.time()
+    if ip not in _rate_limit_store:
+        _rate_limit_store[ip] = []
+    # Clean old entries
+    _rate_limit_store[ip] = [t for t in _rate_limit_store[ip] if now - t < _RATE_LIMIT_WINDOW]
+    if len(_rate_limit_store[ip]) >= _RATE_LIMIT_MAX:
+        return False
+    _rate_limit_store[ip].append(now)
+    return True
+
+
+def _get_cache_key(text):
+    """Generate cache key from text."""
+    return hashlib.md5(text.encode('utf-8')).hexdigest()
+
+
+def _get_cached_response(text):
+    """Get cached response if exists and not expired."""
+    key = _get_cache_key(text)
+    if key in _analyze_cache:
+        data, ts = _analyze_cache[key]
+        if time.time() - ts < _ANALYZE_CACHE_TTL:
+            _analyze_cache.move_to_end(key)
+            return data
+        else:
+            del _analyze_cache[key]
+    return None
+
+
+def _set_cached_response(text, response_data):
+    """Store response in cache."""
+    key = _get_cache_key(text)
+    _analyze_cache[key] = (response_data, time.time())
+    # Evict oldest if over limit
+    while len(_analyze_cache) > _ANALYZE_CACHE_MAX:
+        _analyze_cache.popitem(last=False)
+
+
 # Global model instances
 summarization_model = None
 spelling_model = None
@@ -1069,6 +1185,14 @@ def _is_orthographic_variant(word1: str, word2: str) -> bool:
 
 @app.route('/api/analyze', methods=['POST'])
 def analyze_text():
+    # ── Rate Limiting (P3) ──
+    client_ip = request.headers.get('X-Forwarded-For', request.remote_addr)
+    if not _check_rate_limit(client_ip):
+        return jsonify({
+            'error': 'Rate limit exceeded. Please wait before making more requests.',
+            'status': 'error'
+        }), 429
+
     """
     Perform sequential analysis (Spelling -> Grammar -> Punctuation) 
     and return word-level suggestions with offsets.
@@ -1090,6 +1214,12 @@ def analyze_text():
         if not text:
             return jsonify({'error': 'Text is required', 'status': 'error'}), 400
 
+        # ── Cache Check (P3) ──
+        cached = _get_cached_response(text)
+        if cached:
+            logger.info(f"[ANALYZE] Cache hit for text (len={len(text)})")
+            return jsonify(cached)
+
         # Reject inputs that are predominantly non-Arabic (code, markup, etc.)
         arabic_chars = len(re.findall(r'[\u0600-\u06FF]', text))
         alpha_chars = len(re.findall(r'[a-zA-Z\u0600-\u06FF]', text))
@@ -1358,6 +1488,22 @@ def analyze_text():
         except Exception as e:
             logger.error(f"[ANALYZE] Hamza fix failed: {type(e).__name__}: {e}")
 
+        # ── Ta Marbuta fix pass (P2) ──
+        # Catches common ه→ة errors like المدرسه→المدرسة at pipeline level.
+        try:
+            ta_fixed, ta_changes = _fix_ta_marbuta(current_text)
+            if ta_fixed != current_text:
+                for tc in ta_changes:
+                    ctx.add_patch(
+                        'spelling', tc['start'], tc['end'],
+                        tc['correction'], confidence=0.95,
+                    )
+                    logger.info(f"[TA-MARBUTA] '{tc['original']}' → '{tc['correction']}'")
+                ctx.mutate_text(ta_fixed, OffsetMapper)
+                current_text = ctx.current_text
+        except Exception as e:
+            logger.error(f"[ANALYZE] Ta Marbuta fix failed: {type(e).__name__}: {e}")
+
         # 2. Grammar (runs on spelling-corrected text — word-level dependency)
         try:
             t0 = time.time()
@@ -1605,6 +1751,10 @@ def analyze_text():
         if stage_errors:
             response_data['warnings'] = stage_errors
 
+        # ── Cache Store (P3) ──
+        if response_status == 'success':
+            _set_cached_response(text, response_data)
+
         return jsonify(response_data)
 
     except Exception as e:
@@ -1617,6 +1767,52 @@ def analyze_text():
         }), 500
 
 
+@app.route('/api/feedback', methods=['POST'])
+def submit_feedback():
+    """Accept user feedback on correction suggestions."""
+    try:
+        if not request.is_json:
+            return jsonify({'error': 'Request must be JSON', 'status': 'error'}), 400
+
+        data = request.get_json()
+        suggestion_id = data.get('suggestion_id', '')
+        helpful = data.get('helpful', None)
+        text = data.get('text', '')[:200]  # Truncate for safety
+        original = data.get('original', '')[:100]
+        correction = data.get('correction', '')[:100]
+
+        if helpful is None:
+            return jsonify({'error': 'helpful field is required', 'status': 'error'}), 400
+
+        # Log feedback (simple file-based for now)
+        feedback_entry = {
+            'timestamp': time.strftime('%Y-%m-%d %H:%M:%S'),
+            'suggestion_id': suggestion_id,
+            'helpful': helpful,
+            'original': original,
+            'correction': correction,
+            'text_snippet': text,
+            'ip': request.headers.get('X-Forwarded-For', request.remote_addr),
+        }
+        logger.info(f"[FEEDBACK] {feedback_entry}")
+
+        # Append to feedback log file
+        try:
+            feedback_dir = Path(__file__).parent.parent / 'logs'
+            feedback_dir.mkdir(exist_ok=True)
+            with open(feedback_dir / 'feedback.jsonl', 'a', encoding='utf-8') as f:
+                import json
+                f.write(json.dumps(feedback_entry, ensure_ascii=False) + '\n')
+        except Exception as log_err:
+            logger.warning(f"[FEEDBACK] Could not write to file: {log_err}")
+
+        return jsonify({'status': 'success', 'message': 'شكراً لملاحظاتك!'})
+
+    except Exception as e:
+        logger.error(f"[FEEDBACK] Error: {e}")
+        return jsonify({'error': 'Failed to submit feedback', 'status': 'error'}), 500
+
+
 @app.errorhandler(404)
 def not_found(error):
     """Handle 404 errors."""

src/css/components.css

@@ -796,6 +796,87 @@
   margin-top: var(--spacing-sm);
 }
 
+/* ── Type Icon ── */
+.popover-type-icon {
+  font-size: 16px;
+  line-height: 1;
+}
+
+/* ── Original Label ── */
+#tooltip-original {
+  text-decoration: none;
+  color: var(--color-text-secondary);
+  margin-bottom: var(--spacing-sm);
+  font-size: var(--font-size-sm);
+}
+
+.popover-original-label {
+  font-weight: var(--font-weight-bold);
+  color: var(--color-text-muted);
+  font-size: var(--font-size-label);
+}
+
+.popover-original-word {
+  text-decoration: line-through;
+  color: var(--color-error);
+  font-weight: var(--font-weight-semibold);
+}
+
+/* ── Feedback Buttons ── */
+.popover-feedback {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  margin-top: 8px;
+  padding-top: 8px;
+  border-top: 1px solid var(--color-border);
+  justify-content: center;
+}
+
+.popover-feedback-label {
+  font-size: var(--font-size-label);
+  color: var(--color-text-muted);
+}
+
+.popover-feedback-btn {
+  width: 32px;
+  height: 32px;
+  border: 1px solid var(--color-border);
+  border-radius: 50%;
+  background: var(--color-surface);
+  cursor: pointer;
+  font-size: 14px;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  transition: all 0.2s ease;
+}
+
+.popover-feedback-yes:hover {
+  background: rgba(34, 197, 94, 0.15);
+  border-color: #22c55e;
+  transform: scale(1.1);
+}
+
+.popover-feedback-no:hover {
+  background: rgba(239, 68, 68, 0.15);
+  border-color: #ef4444;
+  transform: scale(1.1);
+}
+
+.popover-feedback-thanks {
+  font-size: var(--font-size-label);
+  color: #22c55e;
+  font-weight: var(--font-weight-bold);
+  animation: fadeIn 0.3s ease;
+}
+
+@keyframes fadeIn {
+  from { opacity: 0; transform: translateY(4px); }
+  to { opacity: 1; transform: translateY(0); }
+}
+
+
 /* ── Alt chips in suggestion cards ── */
 .suggestion-card-alts {
   display: flex;

src/js/editor.js

@@ -363,18 +363,19 @@ function showTooltip(element) {
   const alternativesEl = document.getElementById('tooltip-alternatives');
 
   const typeMap = {
-    spelling: 'خطأ إملائي',
-    grammar: 'خطأ نحوي',
-    punctuation: 'علامات ترقيم'
+    spelling: { label: 'خطأ إملائي', icon: '🔤' },
+    grammar: { label: 'خطأ نحوي', icon: '📝' },
+    punctuation: { label: 'علامات ترقيم', icon: '✏️' }
   };
 
   if (typeEl) {
-    typeEl.textContent = typeMap[suggestion.type] || suggestion.type;
+    const typeInfo = typeMap[suggestion.type] || { label: suggestion.type, icon: '📋' };
+    typeEl.innerHTML = `<span class="popover-type-icon">${typeInfo.icon}</span> ${typeInfo.label}`;
     typeEl.className = `popover-type popover-type--${suggestion.type}`;
   }
 
   if (originalEl) {
-    originalEl.textContent = suggestion.original;
+    originalEl.innerHTML = `<span class="popover-original-label">الأصل:</span> <span class="popover-original-word">${escapeHtml(suggestion.original)}</span>`;
   }
 
   // Render alternatives
@@ -392,25 +393,38 @@ function showTooltip(element) {
       if (isKeep) return; // render keep button last
       const isMain = i === 0;
       const btnClass = isMain ? 'popover-alt-btn popover-alt-main' : 'popover-alt-btn';
-      html += `<button class="${btnClass}" data-alt-correction="${escapeHtml(alt)}" type="button">${escapeHtml(alt)}</button>`;
+      html += `<button class="${btnClass}" data-alt-correction="${escapeHtml(alt)}" type="button">${isMain ? '✓ ' : ''}${escapeHtml(alt)}</button>`;
     });
     // Render keep button at end
     html += `<button class="popover-alt-btn popover-alt-keep" data-alt-correction="${escapeHtml(suggestion.original)}" type="button">إبقاء كما هي</button>`;
+    // Render feedback buttons
+    html += `<div class="popover-feedback">
+      <span class="popover-feedback-label">هل الاقتراح مفيد؟</span>
+      <button class="popover-feedback-btn popover-feedback-yes" data-feedback="yes" type="button" title="مفيد">👍</button>
+      <button class="popover-feedback-btn popover-feedback-no" data-feedback="no" type="button" title="غير مفيد">👎</button>
+    </div>`;
     alternativesEl.innerHTML = html;
 
-    // Bind click events
+    // Bind click events for alternatives
     alternativesEl.querySelectorAll('.popover-alt-btn').forEach(btn => {
       btn.addEventListener('click', () => {
         const correctionText = btn.dataset.altCorrection;
         if (correctionText === suggestion.original) {
-          // "Keep as-is" — just dismiss the suggestion
           dismissSuggestion(suggestion);
         } else {
-          // Apply this alternative correction
           applyAlternativeCorrection(suggestion, correctionText);
         }
       });
     });
+
+    // Bind feedback buttons
+    alternativesEl.querySelectorAll('.popover-feedback-btn').forEach(btn => {
+      btn.addEventListener('click', () => {
+        const helpful = btn.dataset.feedback === 'yes';
+        _sendFeedback(suggestion, helpful);
+        btn.closest('.popover-feedback').innerHTML = '<span class="popover-feedback-thanks">شكراً لملاحظاتك! ✓</span>';
+      });
+    });
   }
 
   const rect = element.getBoundingClientRect();
@@ -520,6 +534,9 @@ function applySuggestionAtOffsets(suggestion) {
   } finally {
     _isApplyingSuggestion = false;
   }
+  // P2/User Request: Auto re-analyze after applying suggestion
+  // Calls analyzeText() DIRECTLY (not delayed) for instant re-analysis.
+  setTimeout(() => { analyzeText(); }, 300);
 }
 
 function applyCorrection() {
@@ -603,6 +620,8 @@ function applyAlternativeCorrection(suggestion, correctionText) {
   } finally {
     _isApplyingSuggestion = false;
   }
+  // P2/User Request: Auto re-analyze after applying alternative correction
+  setTimeout(() => { analyzeText(); }, 300);
 }
 
 function dismissSuggestion(suggestion) {
@@ -682,6 +701,8 @@ function applyAllSuggestions() {
   } finally {
     _isApplyingSuggestion = false;
   }
+  // P2/User Request: Auto re-analyze after applying all suggestions
+  setTimeout(() => { analyzeText(); }, 300);
 }
 
 function clearEditor() {
@@ -745,6 +766,22 @@ function copyText() {
   });
 }
 
+// ── Feedback API (P2) ──
+function _sendFeedback(suggestion, helpful) {
+  const apiBase = window.BAYAN_API_BASE || '';
+  fetch(`${apiBase}/api/feedback`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      suggestion_id: suggestion.id || '',
+      helpful: helpful,
+      original: suggestion.original || '',
+      correction: suggestion.correction || '',
+      text: (document.getElementById('editor-container')?.textContent || '').substring(0, 200),
+    })
+  }).catch(err => console.warn('[Feedback] Failed:', err));
+}
+
 if (typeof module !== 'undefined' && module.exports) {
   module.exports = {
     initEditor,

tests/deep_dive_expanded.json

@@ -0,0 +1,1323 @@
+{
+  "timestamp": "2026-06-20T19:17:40.208323+00:00",
+  "api_base": "https://bayan10-bayan-api.hf.space",
+  "health": {
+    "environment": "huggingface_spaces",
+    "mode": "hf_spaces_local",
+    "models": {
+      "autocomplete": true,
+      "grammar": true,
+      "punctuation": true,
+      "spelling": true,
+      "summarization": true
+    },
+    "note": "Free tier: summarization local, other models return input unchanged",
+    "status": "healthy",
+    "supabase": {
+      "configured": true
+    }
+  },
+  "cat1": [
+    {
+      "id": "C1-01",
+      "category": 1,
+      "input": "كانت الفتيات يلعبون في الحديقه",
+      "a_spelling": "كانت الفتيات يلعبون في الحديقة",
+      "a_grammar_on_original": "كانت الفتيات يلعبن في الحديقة",
+      "a_grammar_on_spell_corrected": "كانت الفتيات يلعبن في الحديقة",
+      "a_punctuation": "كانت الفتيات يلعبون في الحديقه.",
+      "grammar_diff_orig_vs_spell": [],
+      "b_corrected": "كانت الفتيات يلعبن في الحديقة.",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "يلعبن",
+          "end": 19,
+          "id": "48d68e84-19e8-4af0-b1ea-b7ae4c7427f4",
+          "locked": true,
+          "original": "يلعبون",
+          "priority": 3,
+          "start": 13,
+          "type": "grammar"
+        },
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "الحديقة.",
+          "end": 30,
+          "id": "0012d2dc-08f6-44ad-a9d1-e07230045474",
+          "locked": true,
+          "original": "الحديقه",
+          "priority": 2,
+          "start": 23,
+          "type": "punctuation"
+        }
+      ]
+    },
+    {
+      "id": "C1-02",
+      "category": 1,
+      "input": "ان الطالبات ذهبو الى الجامعه",
+      "a_spelling": "ان الط ابت ذهبوا إلى الجامعه",
+      "a_grammar_on_original": "إن الطالبات ذهبن إلى الجامعة",
+      "a_grammar_on_spell_corrected": "إن الطلاب ذهبوا إلى الجامعة",
+      "a_punctuation": "ان الطالبات ذهبو الى الجامعه!",
+      "grammar_diff_orig_vs_spell": [
+        {
+          "word_idx": 1,
+          "gram_on_orig": "الطالبات",
+          "gram_on_spell": "الطلاب"
+        },
+        {
+          "word_idx": 2,
+          "gram_on_orig": "ذهبن",
+          "gram_on_spell": "ذهبوا"
+        }
+      ],
+      "b_corrected": "إن الطالبات ذهبن ذهبوا الجامعة.",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "ذهبن",
+          "end": 16,
+          "id": "8a21b71d-8a87-4e29-a828-4ce2b343ae2a",
+          "locked": true,
+          "original": "ذهبو",
+          "priority": 3,
+          "start": 12,
+          "type": "grammar"
+        },
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "الجامعة.",
+          "end": 28,
+          "id": "0c9e127e-8e64-4cc4-a928-03651da1dd15",
+          "locked": true,
+          "original": "الجامعه",
+          "priority": 2,
+          "start": 21,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "إن",
+          "end": 2,
+          "id": "156cd1ce-37cd-4ade-888c-9e8d12a83b05",
+          "locked": true,
+          "original": "ان",
+          "priority": 1,
+          "start": 0,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [
+            "ذهبوا",
+            "ال",
+            "الم",
+            "الى"
+          ],
+          "confidence": 0.9,
+          "correction": "ذهبوا",
+          "end": 20,
+          "id": "836d7346-3ea8-4851-bc69-53df3e1ff6b4",
+          "locked": true,
+          "original": "الى",
+          "priority": 1,
+          "start": 17,
+          "type": "spelling"
+        }
+      ]
+    },
+    {
+      "id": "C1-03",
+      "category": 1,
+      "input": "هذة المدينه جميله جدا ومناخها معتدل",
+      "a_spelling": "هذة المدينه جميله جدا ومناخها معتدل",
+      "a_grammar_on_original": "هذه المدينة جميلة جدا ومناخها معتدل",
+      "a_grammar_on_spell_corrected": "هذه المدينة جميلة جدا ومناخها معتدل",
+      "a_punctuation": "هذة المدينه جميله جدا ومناخها معتدل.",
+      "grammar_diff_orig_vs_spell": [],
+      "b_corrected": "هذه المدينة جميلة جدا ومناخها معتدل.",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "معتدل.",
+          "end": 35,
+          "id": "b960d84a-bcf6-40c3-976e-29632ad7f302",
+          "locked": true,
+          "original": "معتدل",
+          "priority": 2,
+          "start": 30,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "هذه المدينة جميلة",
+          "end": 17,
+          "id": "a89acaad-669a-4867-9ebd-6bd5cbfad2ea",
+          "locked": true,
+          "original": "هذة المدينه جميله",
+          "priority": 1,
+          "start": 0,
+          "type": "spelling"
+        }
+      ]
+    },
+    {
+      "id": "C1-04",
+      "category": 1,
+      "input": "الطلاب اجتهدو في دراستهم وحققو نتائج ممتازه",
+      "a_spelling": "الطلاب الاجتهادوا في دراستهم وحققوا نتائج ممتازه",
+      "a_grammar_on_original": "الطلاب اجتهدو في دراستهم وحققوا نتائج ممتازة",
+      "a_grammar_on_spell_corrected": "الطلاب اجتهدوا في دراستهم وحققوا نتائج ممتازة",
+      "a_punctuation": "الطلاب اجتهدو في دراستهم وحققو نتائج ممتازه.",
+      "grammar_diff_orig_vs_spell": [
+        {
+          "word_idx": 1,
+          "gram_on_orig": "اجتهدو",
+          "gram_on_spell": "اجتهدوا"
+        }
+      ],
+      "b_corrected": "الطلاب اجتهدو في دراستهم وحققوا نتائج ممتازة.",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "ممتازة.",
+          "end": 43,
+          "id": "19a6216b-b7a1-4c0f-acd9-5c698617443b",
+          "locked": true,
+          "original": "ممتازه",
+          "priority": 2,
+          "start": 37,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [
+            "وحققوا",
+            "وحقوق",
+            "وحقق",
+            "وحققو"
+          ],
+          "confidence": 0.9,
+          "correction": "وحققوا",
+          "end": 30,
+          "id": "cdcf61bd-8983-44c8-9c3d-8792dc8027c3",
+          "locked": true,
+          "original": "وحققو",
+          "priority": 1,
+          "start": 25,
+          "type": "spelling"
+        }
+      ]
+    },
+    {
+      "id": "C1-05",
+      "category": 1,
+      "input": "ذهب الولد الى المكتبه وقرا كتاب مفيد",
+      "a_spelling": "ذهب الولد إلى المكتبة وقرا كتاب مفيد",
+      "a_grammar_on_original": "ذهب الولد إلى المكتبة وقرا كتابا مفيدا",
+      "a_grammar_on_spell_corrected": "ذهب الولد إلى المكتبة وقرا كتابا مفيدا",
+      "a_punctuation": "ذهب الولد الى المكتبه وقرا، كتاب مفيد",
+      "grammar_diff_orig_vs_spell": [],
+      "b_corrected": "ذهب الولد إلى المكتبة وقرا كتابا مفيدا",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "كتابا مفيدا",
+          "end": 36,
+          "id": "ede929d9-8112-4d65-a460-7a44cee535f5",
+          "locked": true,
+          "original": "كتاب مفيد",
+          "priority": 3,
+          "start": 27,
+          "type": "grammar"
+        },
+        {
+          "alternatives": [
+            "إلى",
+            "ال",
+            "الم",
+            "الى"
+          ],
+          "confidence": 0.9,
+          "correction": "إلى",
+          "end": 13,
+          "id": "affb4882-0466-4184-93e7-fb3463132a83",
+          "locked": true,
+          "original": "الى",
+          "priority": 1,
+          "start": 10,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [
+            "المكتبة",
+            "المكتب",
+            "المشتبه",
+            "المكتبه"
+          ],
+          "confidence": 0.9,
+          "correction": "المكتبة",
+          "end": 21,
+          "id": "3f73b099-d428-4c05-98f8-138fb1d83c54",
+          "locked": true,
+          "original": "المكتبه",
+          "priority": 1,
+          "start": 14,
+          "type": "spelling"
+        }
+      ]
+    }
+  ],
+  "cat7": [
+    {
+      "id": "C7-01",
+      "category": 7,
+      "input": "ذهب الولد الى المدرسه وقابل المعلمه واخذ الكتاب",
+      "desc": "3-stage chain: spelling الى→إلى, grammar المدرسه→المدرسة, punc adds marks",
+      "a_spelling": "ذهب الولد إلى المدرسه وقابل المعلمه وأخذ الكتاب",
+      "a_grammar": "ذهب الولد إلى المدرسة وقابل المعلمة وأخذ الكتاب",
+      "a_punc": "ذهب الولد الى المدرسه وقابل، المعلمه واخذ الكتاب،",
+      "b_corrected": "ذهب الولد إلى المدرسة وقابل المعلمة وأخ�� الكتاب.",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "الكتاب.",
+          "end": 47,
+          "id": "3e740303-1dcd-42ec-bd6c-7f0af8069e44",
+          "locked": true,
+          "original": "الكتاب",
+          "priority": 2,
+          "start": 41,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "المدرسة",
+          "end": 21,
+          "id": "84f953ae-2d0a-4e99-a07d-7d35638ba843",
+          "locked": true,
+          "original": "المدرسه",
+          "priority": 1,
+          "start": 14,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "المعلمة",
+          "end": 35,
+          "id": "2e793ab2-29fc-454b-8490-ca7cfdfe4404",
+          "locked": true,
+          "original": "المعلمه",
+          "priority": 1,
+          "start": 28,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [
+            "إلى",
+            "ال",
+            "الم",
+            "الى"
+          ],
+          "confidence": 0.9,
+          "correction": "إلى",
+          "end": 13,
+          "id": "d1b951c7-f94e-4941-986c-8ce5fa51bab0",
+          "locked": true,
+          "original": "الى",
+          "priority": 1,
+          "start": 10,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [
+            "وأخذ",
+            "والذ",
+            "واخت",
+            "واخذ"
+          ],
+          "confidence": 0.9,
+          "correction": "وأخذ",
+          "end": 40,
+          "id": "c024f6ae-4740-4d87-9a0d-95818699e4f3",
+          "locked": true,
+          "original": "واخذ",
+          "priority": 1,
+          "start": 36,
+          "type": "spelling"
+        }
+      ],
+      "b_suggestion_count": 5,
+      "overlapping_suggestions": []
+    },
+    {
+      "id": "C7-02",
+      "category": 7,
+      "input": "كانت البنات يلعبون في الحديقه الجميله وفجأه سقطت احداهن",
+      "desc": "Multiple overlapping corrections across all stages",
+      "a_spelling": "كانت البنات يلعبون في الحديقه الجميله وفجأه سقطت احداهن",
+      "a_grammar": "كانت البنات يلعبن في الحديقة الجميلة وفجأة سقطت أختهن",
+      "a_punc": "كانت البنات يلعبون في الحديقه الجميله وفجأه، سقطت احداهن",
+      "b_corrected": "كانت البنات يلعبن في الحديقة الجميلة وفجأة سقطت أختهن.",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "يلعبن",
+          "end": 18,
+          "id": "2d1e033f-60d9-4921-bb65-d13b890f44d4",
+          "locked": true,
+          "original": "يلعبون",
+          "priority": 3,
+          "start": 12,
+          "type": "grammar"
+        },
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "أختهن.",
+          "end": 55,
+          "id": "409bca6e-33d5-4339-8d75-d8dc1e3a8cea",
+          "locked": true,
+          "original": "احداهن",
+          "priority": 2,
+          "start": 49,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "الحديقة الجميلة وفجأة",
+          "end": 43,
+          "id": "ac5b5008-0563-4168-99aa-46c8de59d1d1",
+          "locked": true,
+          "original": "الحديقه الجميله وفجأه",
+          "priority": 1,
+          "start": 22,
+          "type": "spelling"
+        }
+      ],
+      "b_suggestion_count": 3,
+      "overlapping_suggestions": []
+    },
+    {
+      "id": "C7-03",
+      "category": 7,
+      "input": "ان الذكاء الاصطناعي يلعب دورا هاما في تطوير التكنولوجيا الحديثه ولذالك يجب الاهتمام بة",
+      "desc": "Long sentence with corrections from all 3 stages",
+      "a_spelling": "أن الذكاء الاصطناعي يلعب دورا هاما في تطوير التكنولوجيا الحديثة ولذا ذلك يجب الاهتمام بة",
+      "a_grammar": "إن الذكاء الاصطناعي يؤدي دورا مهمّا في تطوير التكنولوجيا الحديثة ولذلك يجب الاهتمام به",
+      "a_punc": "ان الذكاء الاصطناعي يلعب دورا هاما في تطوير التكنولوجيا الحديثه ولذالك؛ يجب الاهتمام بة",
+      "b_corrected": "أن الذكاء الاصطناعي يلعب دورا هاما في تطوير التكنولوجيا الحديثة ولذا ذلك يجب الاهتمام به",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "به",
+          "end": 86,
+          "id": "0a44f7d9-1554-428c-809a-706631ffebdd",
+          "locked": true,
+          "original": "بة",
+          "priority": 1,
+          "start": 84,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [
+            "أن",
+            "ال",
+            "من",
+            "ان"
+          ],
+          "confidence": 0.9,
+          "correction": "أن",
+          "end": 2,
+          "id": "512b3c30-eadb-40fc-a481-1d4cd0909459",
+          "locked": true,
+          "original": "ان",
+          "priority": 1,
+          "start": 0,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [
+            "الحديثة",
+            "الحديث",
+            "الحديثي",
+            "الحديثه"
+          ],
+          "confidence": 0.9,
+          "correction": "الحديثة",
+          "end": 63,
+          "id": "9eb1f571-7852-4813-975d-f7cd79102ec8",
+          "locked": true,
+          "original": "الحديثه",
+          "priority": 1,
+          "start": 56,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [
+            "ولذا ذلك",
+            "ولذالك"
+          ],
+          "confidence": 0.85,
+          "correction": "ولذا ذلك",
+          "end": 70,
+          "id": "266db5db-ed2a-49be-accb-76db9e07697f",
+          "locked": true,
+          "original": "ولذالك",
+          "priority": 1,
+          "start": 64,
+          "type": "spelling"
+        }
+      ],
+      "b_suggestion_count": 4,
+      "overlapping_suggestions": []
+    },
+    {
+      "id": "C7-04",
+      "category": 7,
+      "input": "هذة المدينه جميله جدا ومناخها معتدل طوال العام وسكانها طيبون جدا",
+      "desc": "Multiple ه→ة fixes: does grammar lock prevent punc from adding marks near those words?",
+      "a_spelling": "هذة المدينه جميله جدا ومناخها معتدل طوال العام وسكان طيبون جدا",
+      "a_grammar": "هذه المدينة جميلة جدا ومناخها معتدل طوال العام وسكانها طيبون جدا",
+      "a_punc": "هذة المدينه جميله جدا ومناخها معتدل طوال العام وسكانها طيبون جدا.",
+      "b_corrected": "هذه المدينة جميلة جدا ومناخها معتدل طوال العام وسكانها طيبون جدا.",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "جدا.",
+          "end": 64,
+          "id": "0dd9d98a-f146-492b-87c7-dba4913bdfd4",
+          "locked": true,
+          "original": "جدا",
+          "priority": 2,
+          "start": 61,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "هذه المدينة جميلة",
+          "end": 17,
+          "id": "e12aa2f9-63d7-4f10-a128-13b0b1bbee9a",
+          "locked": true,
+          "original": "هذة المدينه جميله",
+          "priority": 1,
+          "start": 0,
+          "type": "spelling"
+        }
+      ],
+      "b_suggestion_count": 2,
+      "overlapping_suggestions": []
+    },
+    {
+      "id": "C7-05",
+      "category": 7,
+      "input": "الطلاب اللذين اجتهدو في دراستهم حققو نتائج ممتازه في الأمتحانات الصعبه",
+      "desc": "Heavy corrections needed across stages",
+      "a_spelling": "الطلاب اللذين اجتهد في دراستهم حقوق نتائج ممتازه في الأمتحانات الصعبه",
+      "a_grammar": "الطلاب الذين اجتهدو في دراستهم حققوا نتائج ممتازة في الامتحانات الصعبة",
+      "a_punc": "الطلاب اللذين اجتهدو في دراستهم حققو نتائج ممتازه في الأمتحانات الصعبه.",
+      "b_corrected": "الطلاب اللذين اجتهد في دراستهم حققوا نتائج ممتازة في الأمتحانات الصعبة.",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "حققوا",
+          "end": 36,
+          "id": "b2ae6a56-0879-4572-837f-875895de9020",
+          "locked": true,
+          "original": "حققو",
+          "priority": 3,
+          "start": 32,
+          "type": "grammar"
+        },
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "الصعبة.",
+          "end": 70,
+          "id": "b075d0de-0e6d-4d88-a897-8e79e1845116",
+          "locked": true,
+          "original": "الصعبه",
+          "priority": 2,
+          "start": 64,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "ممتازة",
+          "end": 49,
+          "id": "ee092cbc-bc6f-41f0-9c98-7cc2edeee671",
+          "locked": true,
+          "original": "ممتازه",
+          "priority": 1,
+          "start": 43,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [
+            "اجتهد",
+            "اجتهدو"
+          ],
+          "confidence": 0.9,
+          "correction": "اجتهد",
+          "end": 20,
+          "id": "97a2307e-5a5a-4668-a4fe-052bbf86c4d9",
+          "locked": true,
+          "original": "اجتهدو",
+          "priority": 1,
+          "start": 14,
+          "type": "spelling"
+        }
+      ],
+      "b_suggestion_count": 4,
+      "overlapping_suggestions": []
+    }
+  ],
+  "cat8x": [
+    {
+      "id": "C8X-مدرسة-al",
+      "category": 8,
+      "input": "المدرسة",
+      "root": "مدرسة",
+      "prefix_combo": "al",
+      "track_a_spelling": "المدرسة",
+      "changed": false
+    },
+    {
+      "id": "C8X-مدرسة-wal",
+      "category": 8,
+      "input": "والمدرسة",
+      "root": "مدرسة",
+      "prefix_combo": "wal",
+      "track_a_spelling": "والمدرسة في المدرسة",
+      "changed": true
+    },
+    {
+      "id": "C8X-مدرسة-bal",
+      "category": 8,
+      "input": "بالمدرسة",
+      "root": "مدرسة",
+      "prefix_combo": "bal",
+      "track_a_spelling": "بالمدرسة في المدرسة",
+      "changed": true
+    },
+    {
+      "id": "C8X-مدرسة-lal",
+      "category": 8,
+      "input": "للمدرسة",
+      "root": "مدرسة",
+      "prefix_combo": "lal",
+      "track_a_spelling": "للمدرسة",
+      "changed": false
+    },
+    {
+      "id": "C8X-شمس-al",
+      "category": 8,
+      "input": "الشمس",
+      "root": "شمس",
+      "prefix_combo": "al",
+      "track_a_spelling": "الشمس",
+      "changed": false
+    },
+    {
+      "id": "C8X-شمس-wal",
+      "category": 8,
+      "input": "والشمس",
+      "root": "شمس",
+      "prefix_combo": "wal",
+      "track_a_spelling": "والشمس والشمس",
+      "changed": true
+    },
+    {
+      "id": "C8X-شمس-bal",
+      "category": 8,
+      "input": "بالشمس",
+      "root": "شمس",
+      "prefix_combo": "bal",
+      "track_a_spelling": "الشمس",
+      "changed": true
+    },
+    {
+      "id": "C8X-شمس-lal",
+      "category": 8,
+      "input": "للشمس",
+      "root": "شمس",
+      "prefix_combo": "lal",
+      "track_a_spelling": "للشمس",
+      "changed": false
+    },
+    {
+      "id": "C8X-أمة-al",
+      "category": 8,
+      "input": "الأمة",
+      "root": "أمة",
+      "prefix_combo": "al",
+      "track_a_spelling": "الأمة",
+      "changed": false
+    },
+    {
+      "id": "C8X-أمة-wal",
+      "category": 8,
+      "input": "والأمة",
+      "root": "أمة",
+      "prefix_combo": "wal",
+      "track_a_spelling": "والأمة الأمة",
+      "changed": true
+    },
+    {
+      "id": "C8X-أمة-bal",
+      "category": 8,
+      "input": "بالأمة",
+      "root": "أمة",
+      "prefix_combo": "bal",
+      "track_a_spelling": "الأمة",
+      "changed": true
+    },
+    {
+      "id": "C8X-أمة-lal",
+      "category": 8,
+      "input": "للأمة",
+      "root": "أمة",
+      "prefix_combo": "lal",
+      "track_a_spelling": "للأمة",
+      "changed": false
+    },
+    {
+      "id": "C8X-نافذة-al",
+      "category": 8,
+      "input": "النافذة",
+      "root": "نافذة",
+      "prefix_combo": "al",
+      "track_a_spelling": "النافذة",
+      "changed": false
+    },
+    {
+      "id": "C8X-نافذة-wal",
+      "category": 8,
+      "input": "والنافذة",
+      "root": "نافذة",
+      "prefix_combo": "wal",
+      "track_a_spelling": "النافذة",
+      "changed": true
+    },
+    {
+      "id": "C8X-نافذة-bal",
+      "category": 8,
+      "input": "بالنافذة",
+      "root": "نافذة",
+      "prefix_combo": "bal",
+      "track_a_spelling": "النافذة",
+      "changed": true
+    },
+    {
+      "id": "C8X-نافذة-lal",
+      "category": 8,
+      "input": "للنافذة",
+      "root": "نافذة",
+      "prefix_combo": "lal",
+      "track_a_spelling": "النافذة",
+      "changed": true
+    },
+    {
+      "id": "C8X-علم-al",
+      "category": 8,
+      "input": "العلم",
+      "root": "علم",
+      "prefix_combo": "al",
+      "track_a_spelling": "العلم",
+      "changed": false
+    },
+    {
+      "id": "C8X-علم-wal",
+      "category": 8,
+      "input": "والعلم",
+      "root": "علم",
+      "prefix_combo": "wal",
+      "track_a_spelling": "والعلم هو العلم",
+      "changed": true
+    },
+    {
+      "id": "C8X-علم-bal",
+      "category": 8,
+      "input": "بالعلم",
+      "root": "علم",
+      "prefix_combo": "bal",
+      "track_a_spelling": "العلم بالعلم",
+      "changed": true
+    },
+    {
+      "id": "C8X-علم-lal",
+      "category": 8,
+      "input": "للعلم",
+      "root": "علم",
+      "prefix_combo": "lal",
+      "track_a_spelling": "للعلم",
+      "changed": false
+    },
+    {
+      "id": "C8X-اقتصاد-al",
+      "category": 8,
+      "input": "الاقتصاد",
+      "root": "اقتصاد",
+      "prefix_combo": "al",
+      "track_a_spelling": "الاقتصاد",
+      "changed": false
+    },
+    {
+      "id": "C8X-اقتصاد-wal",
+      "category": 8,
+      "input": "والاقتصاد",
+      "root": "اقتصاد",
+      "prefix_combo": "wal",
+      "track_a_spelling": "والاقتصاد",
+      "changed": false
+    },
+    {
+      "id": "C8X-اقتصاد-bal",
+      "category": 8,
+      "input": "بالاقتصاد",
+      "root": "اقتصاد",
+      "prefix_combo": "bal",
+      "track_a_spelling": "بالاقتصاد في الاقتصاد",
+      "changed": true
+    },
+    {
+      "id": "C8X-اقتصاد-lal",
+      "category": 8,
+      "input": "للاقتصاد",
+      "root": "اقتصاد",
+      "prefix_combo": "lal",
+      "track_a_spelling": "للاقتصاد الاقتصادي",
+      "changed": true
+    }
+  ],
+  "cat9x": [
+    {
+      "id": "C9X-01",
+      "category": 9,
+      "input": "إنّ",
+      "context": "isolation",
+      "concern": "stays إنّ",
+      "track_a_spelling": "إن إن",
+      "changed": true
+    },
+    {
+      "id": "C9X-02",
+      "category": 9,
+      "input": "أنّ",
+      "context": "isolation",
+      "concern": "stays أنّ",
+      "track_a_spelling": "أن أن",
+      "changed": true
+    },
+    {
+      "id": "C9X-03",
+      "category": 9,
+      "input": "إنّ العلم نور",
+      "context": "sentence",
+      "concern": "إنّ stays",
+      "track_a_spelling": "إن العلم نور",
+      "changed": true
+    },
+    {
+      "id": "C9X-04",
+      "category": 9,
+      "input": "علمت أنّ الامتحان صعب",
+      "context": "sentence",
+      "concern": "أنّ stays",
+      "track_a_spelling": "علمت أن الامتحان صعب",
+      "changed": true
+    },
+    {
+      "id": "C9X-05",
+      "category": 9,
+      "input": "علي",
+      "context": "isolation",
+      "concern": "could be name علي or على",
+      "track_a_spelling": "علي",
+      "changed": false
+    },
+    {
+      "id": "C9X-06",
+      "category": 9,
+      "input": "ذهب علي إلى المدرسة",
+      "context": "sentence",
+      "concern": "علي is a name here",
+      "track_a_spelling": "ذهب علي إلى المدرسة",
+      "changed": false
+    },
+    {
+      "id": "C9X-07",
+      "category": 9,
+      "input": "جلس علي الكرسي",
+      "context": "sentence",
+      "concern": "AMBIGUOUS: علي=name or على=on",
+      "track_a_spelling": "جلس علي الكرسي",
+      "changed": false
+    }
+  ],
+  "cat10x": [
+    {
+      "id": "C10X-01a",
+      "category": 10,
+      "input": "الحديقه جميلة جدا",
+      "concern": "error_at_start",
+      "track_a_spelling": "الحديقه جميلة جدا",
+      "a_changed": false,
+      "track_b_corrected": "الحديقة جميلة جدا.",
+      "track_b_suggestions": 2
+    },
+    {
+      "id": "C10X-01b",
+      "category": 10,
+      "input": "الجو حار في الحديقه",
+      "concern": "error_at_end",
+      "track_a_spelling": "الجو حار في الحديقة",
+      "a_changed": true,
+      "track_b_corrected": "الجو حار في الحديقة.",
+      "track_b_suggestions": 1
+    },
+    {
+      "id": "C10X-02a",
+      "category": 10,
+      "input": "الى المدرسة ذهب الولد",
+      "concern": "error_at_start",
+      "track_a_spelling": "إلى المدرسة ذهب الولد",
+      "a_changed": true,
+      "track_b_corrected": "إلى المدرسة ذهب الولد.",
+      "track_b_suggestions": 2
+    },
+    {
+      "id": "C10X-02b",
+      "category": 10,
+      "input": "ذهب الولد الى المدرسة",
+      "concern": "error_at_end",
+      "track_a_spelling": "ذهب الولد إلى المدرسة",
+      "a_changed": true,
+      "track_b_corrected": "ذهب الولد إلى المدرسة.",
+      "track_b_suggestions": 2
+    },
+    {
+      "id": "C10X-DRIFT",
+      "category": 10,
+      "input_len": 713,
+      "word_count": 119,
+      "total_suggestions": 16,
+      "front_half_suggestions": 11,
+      "back_half_suggestions": 5,
+      "coordinate_mismatches": [],
+      "suggestions_detail": [
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "إحدىهن وبدأت",
+          "end": 62,
+          "id": "e892df95-0d05-40bd-969a-ccda1305cf2c",
+          "locked": true,
+          "original": "احداهن وبدءت",
+          "priority": 3,
+          "start": 50,
+          "type": "grammar"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "اجتهدوا",
+          "end": 243,
+          "id": "eef5aad7-31f7-4c1e-8095-88dbdda98944",
+          "locked": true,
+          "original": "اجتهدو",
+          "priority": 3,
+          "start": 237,
+          "type": "grammar"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "حققوا",
+          "end": 259,
+          "id": "abfaa89c-119e-4899-9456-6ee78c929298",
+          "locked": true,
+          "original": "حققو",
+          "priority": 3,
+          "start": 255,
+          "type": "grammar"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "",
+          "end": 712,
+          "id": "afcc69ca-f5d8-4907-b85a-e348d0d06a12",
+          "locked": true,
+          "original": "بين الأشجار",
+          "priority": 3,
+          "start": 701,
+          "type": "grammar"
+        },
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "محمد،",
+          "end": 282,
+          "id": "4eef8996-7a31-4d0e-83ca-e05604b975e0",
+          "locked": true,
+          "original": "محمد",
+          "priority": 2,
+          "start": 278,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "جمهورية،",
+          "end": 424,
+          "id": "7e82e486-59af-4002-be9b-5b202dfe8492",
+          "locked": true,
+          "original": "جمهورية",
+          "priority": 2,
+          "start": 417,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "بين،",
+          "end": 497,
+          "id": "6b95fcb5-e190-4dec-8d69-22520c1bb6fe",
+          "locked": true,
+          "original": "بين",
+          "priority": 2,
+          "start": 494,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "الحديقة الجميلة وفجأة",
+          "end": 44,
+          "id": "73493796-1711-4996-9ee4-7013191bc9d8",
+          "locked": true,
+          "original": "الحديقه الجميله وفجأه",
+          "priority": 1,
+          "start": 23,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "بشدة",
+          "end": 72,
+          "id": "13c914ea-5b75-4128-aa42-05576b3d55ae",
+          "locked": true,
+          "original": "بشده",
+          "priority": 1,
+          "start": 68,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "إلى المدرسة",
+          "end": 94,
+          "id": "4e827496-d6f1-4a53-b2f7-c78c2d911195",
+          "locked": true,
+          "original": "الى المدرسه",
+          "priority": 1,
+          "start": 83,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "المعلمة وأخذ",
+          "end": 113,
+          "id": "d7548b59-6379-4b95-a5ed-806e5d1d0cfb",
+          "locked": true,
+          "original": "المعلمه واخذ",
+          "priority": 1,
+          "start": 101,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "أن",
+          "end": 123,
+          "id": "aa9b0140-5740-4343-a3a2-3adfa61fa9d9",
+          "locked": true,
+          "original": "ان",
+          "priority": 1,
+          "start": 121,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "هذه المدينة جميلة",
+          "end": 194,
+          "id": "03378376-a164-46c8-8493-55a0dcd97e3e",
+          "locked": true,
+          "original": "هذة المدينه جميله",
+          "priority": 1,
+          "start": 177,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "ممتازة",
+          "end": 272,
+          "id": "b3aac62f-6a7c-4625-b608-5258fea91fcd",
+          "locked": true,
+          "original": "ممتازه",
+          "priority": 1,
+          "start": 266,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "هذه المحاضرة",
+          "end": 632,
+          "id": "e7994f0f-dc27-4c01-b055-0040683a7643",
+          "locked": true,
+          "original": "هذة المحاضره",
+          "priority": 1,
+          "start": 620,
+          "type": "spelling"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "أهمية",
+          "end": 641,
+          "id": "f68bfac9-17f8-4bbe-9def-1ee35e6ac76a",
+          "locked": true,
+          "original": "اهمية",
+          "priority": 1,
+          "start": 636,
+          "type": "spelling"
+        }
+      ]
+    }
+  ],
+  "cat11": [
+    {
+      "id": "C11-01",
+      "category": 11,
+      "input": "",
+      "desc": "empty_string",
+      "input_len": 0,
+      "crashed": false,
+      "b_corrected": "",
+      "b_suggestions": 0,
+      "error": "HTTP 400: {\"error\":\"Text is required\",\"status\":\"error\"}\n"
+    },
+    {
+      "id": "C11-02",
+      "category": 11,
+      "input": " ",
+      "desc": "whitespace_only",
+      "input_len": 1,
+      "crashed": false,
+      "b_corrected": " ",
+      "b_suggestions": 0,
+      "error": "HTTP 400: {\"error\":\"Text is required\",\"status\":\"error\"}\n"
+    },
+    {
+      "id": "C11-03",
+      "category": 11,
+      "input": "أ",
+      "desc": "single_char",
+      "input_len": 1,
+      "crashed": false,
+      "b_corrected": "أ؟",
+      "b_suggestions": 1,
+      "error": null
+    },
+    {
+      "id": "C11-04",
+      "category": 11,
+      "input": "مستشفياتهم",
+      "desc": "long_single_word",
+      "input_len": 10,
+      "crashed": false,
+      "b_corrected": "في مستشفيات هم",
+      "b_suggestions": 1,
+      "error": null
+    },
+    {
+      "id": "C11-05",
+      "category": 11,
+      "input": "ذهبالولدالىالمدرسةوقابلالمعلمة",
+      "desc": "no_spaces",
+      "input_len": 30,
+      "crashed": false,
+      "b_corrected": "ذهبالولدالىالمدرسةوقابلالمعلمة.",
+      "b_suggestions": 1,
+      "error": null
+    },
+    {
+      "id": "C11-06",
+      "category": 11,
+      "input": "...!؟،،؛؛::...",
+      "desc": "all_punctuation",
+      "input_len": 14,
+      "crashed": false,
+      "b_corrected": ". ! ؟ ، ؛ ::.",
+      "b_suggestions": 1,
+      "error": null
+    },
+    {
+      "id": "C11-07",
+      "category": 11,
+      "input": "(([{هذا النص}]))",
+      "desc": "unbalanced_brackets",
+      "input_len": 16,
+      "crashed": false,
+      "b_corrected": "( ( [ { هذا النص } ] ، و",
+      "b_suggestions": 1,
+      "error": null
+    },
+    {
+      "id": "C11-08",
+      "category": 11,
+      "input": "\"هذا\" 'نص' «اختبار»",
+      "desc": "mixed_quotes",
+      "input_len": 19,
+      "crashed": false,
+      "b_corrected": "\" هذا \" مُنصا ' ' « اختبارا »",
+      "b_suggestions": 1,
+      "error": null
+    },
+    {
+      "id": "C11-09",
+      "category": 11,
+      "input": "يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطن",
+      "desc": "len_299",
+      "input_len": 299,
+      "crashed": false,
+      "b_corrected": "يستخدم الذكاء الاصطناعي تقنيات، التعلم العميق تستخدم الذكاء الاالعميق،ناعي التقنيات التالتعلم،م العميق يستخدم الذكاء الاصطناعية تقنيات التعلم العميقة يستخدم الذكاء الصناعي تقنيات التعلم عميقا يستخدم ا",
+      "b_suggestions": 7,
+      "error": null
+    },
+    {
+      "id": "C11-10",
+      "category": 11,
+      "input": "يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطن",
+      "desc": "len_300",
+      "input_len": 300,
+      "crashed": false,
+      "b_corrected": "يستخدم الذكاء الاصطناعي تقنيات، التعلم العميق تستخدم الذكاء االعميق،صطنالذكاء، التقنيات التعلم العميق يستخدم الذكاء الاصطناعية تقنيات التعلم العميقة يستخدم الذكاء الصناعي تقنيات التعلم عميقا يستخدم ال",
+      "b_suggestions": 14,
+      "error": null
+    },
+    {
+      "id": "C11-11",
+      "category": 11,
+      "input": "يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطناعي تقنيات التعلم العميق يستخدم الذكاء الاصطن",
+      "desc": "len_301",
+      "input_len": 301,
+      "crashed": false,
+      "b_corrected": "يستخدم الذكاء الاصطناعي تقنيات، التعلم العميق تستخدم الذكاء االعميق،صطنالذكاء، التقنيات التعلم العميق يستخدم الذكاء الاصطناعية تقنيات التعلم العميقة يستخدم الذكاء الصناعي تقنيات التعلم عميقا يستخدم ال",
+      "b_suggestions": 14,
+      "error": null
+    },
+    {
+      "id": "C11-12",
+      "category": 11,
+      "input": "يلعب الطلاب في الحديقه بعد المدرسه وقبل العشاء",
+      "desc": "multi_stage_disagreement",
+      "input_len": 46,
+      "crashed": false,
+      "b_corrected": "يلعب الطلاب في الحديقة بعد المدرسة وقبل العشاء.",
+      "b_suggestions": 3,
+      "error": null
+    },
+    {
+      "id": "C11-13",
+      "category": 11,
+      "input": "الحمد لله",
+      "desc": "model_returns_identical",
+      "input_len": 9,
+      "crashed": false,
+      "b_corrected": "الحمد لله.",
+      "b_suggestions": 1,
+      "error": null
+    },
+    {
+      "id": "C11-14",
+      "category": 11,
+      "input": "مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مرحبا مر",
+      "desc": "100x_repeated_word",
+      "input_len": 600,
+      "crashed": false,
+      "b_corrected": "مرحبا مرحبا مرحبا ومرحبا مرحبا مرحبامرحبا مرحبا مرحبا مرحب مرحبا مرحبا وسهلا مرحبا مرحبا ترحيبا مرحبا مرحبا يا مرحبا مرحبا نرحب مرحبا مرحبا ترحيب مرحبا مرحبا أهلا مرحبا مرحبا اهلا مرحبا مرحبا وداعا مر",
+      "b_suggestions": 4,
+      "error": null
+    },
+    {
+      "id": "C11-15",
+      "category": 11,
+      "input": "I went to the مدرسة and met the معلم in the فصل",
+      "desc": "heavy_code_switch",
+      "input_len": 47,
+      "crashed": false,
+      "b_corrected": "I went to the مدرسة and met the معلم in the الفصل.",
+      "b_suggestions": 1,
+      "error": null
+    },
+    {
+      "id": "C11-16",
+      "category": 11,
+      "input": "ايش هالحكي يا زلمة",
+      "desc": "levantine_dialect",
+      "input_len": 18,
+      "crashed": false,
+      "b_corrected": "إيش هالحكي يا زلمة؟",
+      "b_suggestions": 2,
+      "error": null
+    },
+    {
+      "id": "C11-17",
+      "category": 11,
+      "input": "شنو تسوي هسه",
+      "desc": "iraqi_dialect",
+      "input_len": 12,
+      "crashed": false,
+      "b_corrected": "شنو تسوي هسة",
+      "b_suggestions": 1,
+      "error": null
+    },
+    {
+      "id": "C11-RACE",
+      "category": 11,
+      "input": "كانت الفتيات يلعبون في الحديقه",
+      "desc": "parallel_race_condition",
+      "r1_corrected": "كانت الفتيات يلعبن في الحديقة.",
+      "r2_corrected": "كانت الفتيات يلعبن في الحديقة.",
+      "r1_suggestions": 2,
+      "r2_suggestions": 2,
+      "identical": true
+    }
+  ]
+}

tests/deep_dive_expanded.py

@@ -0,0 +1,428 @@
+"""
+BAYAN Deep-Dive Test Harness — EXPANDED (ALL Categories)
+Covers every item from the original prompt that was missing.
+"""
+import sys, os, re, json, time, argparse, concurrent.futures
+from datetime import datetime, timezone
+import requests
+
+API_BASE = "https://bayan10-bayan-api.hf.space"
+TIMEOUT = 60
+
+def api_call(endpoint, text, retries=2):
+    url = f"{API_BASE}{endpoint}"
+    for attempt in range(retries + 1):
+        try:
+            t0 = time.time()
+            resp = requests.post(url, json={"text": text}, timeout=TIMEOUT)
+            elapsed = int((time.time() - t0) * 1000)
+            if resp.status_code == 200:
+                data = resp.json()
+                data['_elapsed_ms'] = elapsed
+                data['_timestamp'] = datetime.now(timezone.utc).isoformat()
+                return data
+            else:
+                if attempt < retries:
+                    time.sleep(2)
+                    continue
+                return {"error": f"HTTP {resp.status_code}: {resp.text[:200]}", "_elapsed_ms": elapsed}
+        except requests.exceptions.Timeout:
+            return {"error": f"Timeout after {TIMEOUT}s"}
+        except Exception as e:
+            return {"error": str(e)}
+
+def track_a_spelling(text):
+    r = api_call("/api/spelling", text)
+    if "error" in r and "corrected_text" not in r:
+        return {"input": text, "output": text, "error": r["error"], "changed": False}
+    c = r.get("corrected_text", text)
+    return {"input": text, "output": c, "changed": c != text, "elapsed_ms": r.get("_elapsed_ms")}
+
+def track_a_grammar(text):
+    r = api_call("/api/grammar", text)
+    if "error" in r and "corrected_text" not in r:
+        return {"input": text, "output": text, "error": r["error"], "changed": False}
+    c = r.get("corrected_text", text)
+    return {"input": text, "output": c, "changed": c != text, "elapsed_ms": r.get("_elapsed_ms"), "timestamp": r.get("_timestamp")}
+
+def track_a_punctuation(text):
+    r = api_call("/api/punctuation", text)
+    if "error" in r and "corrected_text" not in r:
+        return {"input": text, "output": text, "error": r["error"], "changed": False}
+    c = r.get("corrected_text", text)
+    PUNC = '.,;:!?،؛؟'
+    return {"input": text, "output": c, "changed": c != text,
+            "marks_added": sum(1 for ch in c if ch in PUNC) - sum(1 for ch in text if ch in PUNC),
+            "elapsed_ms": r.get("_elapsed_ms")}
+
+def track_b_analyze(text):
+    r = api_call("/api/analyze", text)
+    if "error" in r and "suggestions" not in r:
+        return {"input": text, "error": r["error"], "suggestions": [], "corrected": text}
+    return {
+        "input": text, "original": r.get("original", text),
+        "corrected": r.get("corrected", text),
+        "suggestions": r.get("suggestions", []),
+        "timing_ms": r.get("timing_ms", {}),
+        "elapsed_ms": r.get("_elapsed_ms"),
+    }
+
+def log(msg):
+    print(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}", flush=True)
+
+# ═══════════════════════════════════════════════════════════════════
+# CATEGORY 1 — Cross-model mismatch
+# ═══════════════════════════════════════════════════════════════════
+def run_cat1():
+    log("=== CATEGORY 1: Cross-model mismatch ===")
+    results = []
+    inputs = [
+        {"id": "C1-01", "input": "كانت الفتيات يلعبون في الحديقه"},
+        {"id": "C1-02", "input": "ان الطالبات ذهبو الى الجامعه"},
+        {"id": "C1-03", "input": "هذة المدينه جميله جدا ومناخها معتدل"},
+        {"id": "C1-04", "input": "الطلاب اجتهدو في دراستهم وحققو نتائج ممتازه"},
+        {"id": "C1-05", "input": "ذهب الولد الى المكتبه وقرا كتاب مفيد"},
+    ]
+    for test in inputs:
+        log(f"  {test['id']}: {test['input'][:50]}...")
+        # Track A: each model on ORIGINAL independently
+        a_spell = track_a_spelling(test['input'])
+        a_gram_on_orig = track_a_grammar(test['input'])
+        # NEW: grammar on SPELLING-CORRECTED text
+        a_gram_on_spell = track_a_grammar(a_spell['output'])
+        a_punc = track_a_punctuation(test['input'])
+        # Track B
+        b = track_b_analyze(test['input'])
+
+        # Diff: grammar on original vs grammar on spell-corrected
+        gram_orig_words = a_gram_on_orig['output'].split()
+        gram_spell_words = a_gram_on_spell['output'].split()
+        gram_diff = []
+        for i, (w1, w2) in enumerate(zip(gram_orig_words, gram_spell_words)):
+            if w1 != w2:
+                gram_diff.append({"word_idx": i, "gram_on_orig": w1, "gram_on_spell": w2})
+
+        result = {
+            "id": test['id'], "category": 1, "input": test['input'],
+            "a_spelling": a_spell['output'],
+            "a_grammar_on_original": a_gram_on_orig['output'],
+            "a_grammar_on_spell_corrected": a_gram_on_spell['output'],
+            "a_punctuation": a_punc['output'],
+            "grammar_diff_orig_vs_spell": gram_diff,
+            "b_corrected": b.get('corrected', ''),
+            "b_suggestions": b.get('suggestions', []),
+        }
+        log(f"    A_spell: {a_spell['output'][:60]}")
+        log(f"    A_gram(orig): {a_gram_on_orig['output'][:60]}")
+        log(f"    A_gram(spell): {a_gram_on_spell['output'][:60]}")
+        log(f"    Grammar diff: {gram_diff}")
+        log(f"    B_final: {b.get('corrected','')[:60]}")
+        results.append(result)
+    return results
+
+# ═══════════════════════════════════════════════════════════════════
+# CATEGORY 7 — StageLocker adversarial tests
+# ═══════════════════════════════════════════════════════════════════
+def run_cat7():
+    log("=== CATEGORY 7: StageLocker directionality ===")
+    results = []
+    # 3+ chained mutations: spelling changes length, grammar changes length, punc adds marks
+    inputs = [
+        {"id": "C7-01", "input": "ذهب الولد الى المدرسه وقابل المعلمه واخذ الكتاب",
+         "desc": "3-stage chain: spelling الى→إلى, grammar المدرسه→المدرسة, punc adds marks"},
+        {"id": "C7-02", "input": "كانت البنات يلعبون في الحديقه الجميله وفجأه سقطت احداهن",
+         "desc": "Multiple overlapping corrections across all stages"},
+        {"id": "C7-03", "input": "ان الذكاء الاصطناعي يلعب دورا هاما في تطوير التكنولوجيا الحديثه ولذالك يجب الاهتمام بة",
+         "desc": "Long sentence with corrections from all 3 stages"},
+        {"id": "C7-04", "input": "هذة المدينه جميله جدا ومناخها معتدل طوال العام وسكانها طيبون جدا",
+         "desc": "Multiple ه→ة fixes: does grammar lock prevent punc from adding marks near those words?"},
+        {"id": "C7-05", "input": "الطلاب اللذين اجتهدو في دراستهم حققو نتائج ممتازه في الأمتحانات الصعبه",
+         "desc": "Heavy corrections needed across stages"},
+    ]
+    for test in inputs:
+        log(f"  {test['id']}: {test['input'][:50]}...")
+        a_spell = track_a_spelling(test['input'])
+        a_gram = track_a_grammar(test['input'])
+        a_punc = track_a_punctuation(test['input'])
+        b = track_b_analyze(test['input'])
+
+        # Check: are any suggestions at positions that overlap with corrections from earlier stages?
+        sugg = b.get('suggestions', [])
+        overlaps = []
+        for i, s1 in enumerate(sugg):
+            for j, s2 in enumerate(sugg):
+                if i < j and s1.get('start',0) < s2.get('end',0) and s2.get('start',0) < s1.get('end',0):
+                    overlaps.append({"s1": s1, "s2": s2})
+
+        result = {
+            "id": test['id'], "category": 7, "input": test['input'],
+            "desc": test['desc'],
+            "a_spelling": a_spell['output'],
+            "a_grammar": a_gram['output'],
+            "a_punc": a_punc['output'],
+            "b_corrected": b.get('corrected', ''),
+            "b_suggestions": sugg,
+            "b_suggestion_count": len(sugg),
+            "overlapping_suggestions": overlaps,
+        }
+        log(f"    B_final: {b.get('corrected','')[:60]}")
+        log(f"    Suggestions: {len(sugg)}, Overlaps: {len(overlaps)}")
+        results.append(result)
+    return results
+
+# ═══════════════════════════════════════════════════════════════════
+# CATEGORY 8 EXPANDED — with ال + prefix combos
+# ═══════════════════════════════════════════════════════════════════
+def run_cat8_expanded():
+    log("=== CATEGORY 8 EXPANDED: ال + prefix combos ===")
+    results = []
+    combos = [
+        # root, al_form, wal_form, bal_form, lal_form
+        ("مدرسة", "المدرسة", "والمدرسة", "بالمدرسة", "للمدرسة"),
+        ("شمس", "الشمس", "والشمس", "بالشمس", "للشمس"),
+        ("أمة", "الأمة", "والأمة", "بالأمة", "للأمة"),
+        ("نافذة", "النافذة", "والنافذة", "بالنافذة", "للنافذة"),
+        ("علم", "العلم", "والعلم", "بالعلم", "للعلم"),
+        ("اقتصاد", "الاقتصاد", "والاقتصاد", "بالاقتصاد", "للاقتصاد"),
+    ]
+    for root, al, wal, bal, lal in combos:
+        for label, word in [("al", al), ("wal", wal), ("bal", bal), ("lal", lal)]:
+            a = track_a_spelling(word)
+            result = {
+                "id": f"C8X-{root}-{label}", "category": 8, "input": word,
+                "root": root, "prefix_combo": label,
+                "track_a_spelling": a['output'], "changed": a.get('changed', False),
+            }
+            if a.get('changed'):
+                log(f"  ⚠ C8X-{root}-{label}: '{word}' → '{a['output']}'")
+            results.append(result)
+    return results
+
+# ═══════════════════════════════════════════════════════════════════
+# CATEGORY 9 EXPANDED — missing pairs
+# ═══════════════════════════════════════════════════════════════════
+def run_cat9_expanded():
+    log("=== CATEGORY 9 EXPANDED: Missing confusable pairs ===")
+    results = []
+    tests = [
+        # إنّ / أنّ (with shadda)
+        {"id": "C9X-01", "input": "إنّ", "context": "isolation", "concern": "stays إنّ"},
+        {"id": "C9X-02", "input": "أنّ", "context": "isolation", "concern": "stays أنّ"},
+        {"id": "C9X-03", "input": "إنّ العلم نور", "context": "sentence", "concern": "إنّ stays"},
+        {"id": "C9X-04", "input": "علمت أنّ الامتحان صعب", "context": "sentence", "concern": "أنّ stays"},
+        # على vs علي (name)
+        {"id": "C9X-05", "input": "علي", "context": "isolation", "concern": "could be name علي or على"},
+        {"id": "C9X-06", "input": "ذهب علي إلى المدرسة", "context": "sentence", "concern": "علي is a name here"},
+        {"id": "C9X-07", "input": "جلس علي الكرسي", "context": "sentence", "concern": "AMBIGUOUS: علي=name or على=on"},
+    ]
+    for test in tests:
+        a = track_a_spelling(test['input'])
+        result = {
+            "id": test['id'], "category": 9, "input": test['input'],
+            "context": test['context'], "concern": test['concern'],
+            "track_a_spelling": a['output'], "changed": a.get('changed', False),
+        }
+        if a.get('changed'):
+            log(f"  ⚠ {test['id']}: '{test['input']}' → '{a['output']}' ({test['concern']})")
+        else:
+            log(f"  ✓ {test['id']}: no change")
+        results.append(result)
+    return results
+
+# ═══════════════════════════════════════════════════════════════════
+# CATEGORY 10 EXPANDED — sentence position + 200-word drift test
+# ═══════════════════════════════════════════════════════════════════
+def run_cat10_expanded():
+    log("=== CATEGORY 10 EXPANDED: Position + Cumulative drift ===")
+    results = []
+
+    # Same error at sentence start vs middle
+    log("  Sentence-initial vs mid-sentence:")
+    position_tests = [
+        {"id": "C10X-01a", "input": "الحديقه جميلة جدا", "concern": "error_at_start"},
+        {"id": "C10X-01b", "input": "الجو حار في الحديقه", "concern": "error_at_end"},
+        {"id": "C10X-02a", "input": "الى المدرسة ذهب الولد", "concern": "error_at_start"},
+        {"id": "C10X-02b", "input": "ذهب الولد الى المدرسة", "concern": "error_at_end"},
+    ]
+    for test in position_tests:
+        a = track_a_spelling(test['input'])
+        b = track_b_analyze(test['input'])
+        result = {
+            "id": test['id'], "category": 10, "input": test['input'],
+            "concern": test['concern'],
+            "track_a_spelling": a['output'], "a_changed": a.get('changed', False),
+            "track_b_corrected": b.get('corrected', ''),
+            "track_b_suggestions": len(b.get('suggestions', [])),
+        }
+        log(f"    {test['id']}: A='{a['output'][:40]}' B_sugg={len(b.get('suggestions',[]))}")
+        results.append(result)
+
+    # 200+ word cumulative drift test
+    log("\n  200+ word cumulative drift test:")
+    long_text = (
+        "كانت الفتيات يلعبون في الحديقه الجميله وفجأه سقطت احداهن وبدءت تبكي بشده "
+        "ذهب الولد الى المدرسه وقابل المعلمه واخذ الكتاب "
+        "ان الذكاء الاصطناعي يلعب دورا هاما في تطوير التكنولوجيا "
+        "هذة المدينه جميله جدا ومناخها معتدل طوال العام "
+        "الطلاب الذين اجتهدو في دراستهم حققو نتائج ممتازه "
+        "سافر محمد إلى دبي للعمل في شركة جوجل وقابل أصدقاءه القدامى "
+        "يستخدم الذكاء الاصطناعي تقنيات التعلم العميق في معالجة البيانات الضخمة "
+        "القاهرة عاصمة جمهورية مصر العربية وأكبر مدنها وتقع على ضفاف نهر النيل "
+        "تتراوح درجات الحرارة بين خمس وعشرين وثلاثين درجة مئوية في فصل الصيف "
+        "الحمد لله رب العالمين الرحمن الرحيم مالك يوم الدين "
+        "بسم الله الرحمن الرحيم نبدأ هذة المحاضره عن اهمية التعليم "
+        "يحب الأطفال اللعب في الحديقة وركوب الدراجات والجري بين الأشجار "
+    )
+    word_count = len(long_text.split())
+    log(f"    Input: {word_count} words, {len(long_text)} chars")
+
+    b = track_b_analyze(long_text)
+    sugg = b.get('suggestions', [])
+    # Check coordinates in the back half
+    mid_char = len(long_text) // 2
+    back_half_sugg = [s for s in sugg if s.get('start', 0) >= mid_char]
+    front_half_sugg = [s for s in sugg if s.get('start', 0) < mid_char]
+
+    # Verify coordinates: does original[start:end] == suggestion['original']?
+    coord_mismatches = []
+    for s in sugg:
+        start, end = s.get('start', 0), s.get('end', 0)
+        expected_text = long_text[start:end]
+        actual_text = s.get('original', '')
+        if expected_text != actual_text:
+            coord_mismatches.append({
+                "start": start, "end": end,
+                "expected_from_coords": expected_text,
+                "actual_in_suggestion": actual_text,
+                "correction": s.get('correction', ''),
+                "type": s.get('type', ''),
+            })
+
+    result = {
+        "id": "C10X-DRIFT", "category": 10, "input_len": len(long_text),
+        "word_count": word_count,
+        "total_suggestions": len(sugg),
+        "front_half_suggestions": len(front_half_sugg),
+        "back_half_suggestions": len(back_half_sugg),
+        "coordinate_mismatches": coord_mismatches,
+        "suggestions_detail": sugg,
+    }
+    log(f"    Total suggestions: {len(sugg)} (front: {len(front_half_sugg)}, back: {len(back_half_sugg)})")
+    log(f"    Coordinate mismatches: {len(coord_mismatches)}")
+    for m in coord_mismatches:
+        log(f"      [{m['start']}:{m['end']}] expected='{m['expected_from_coords']}' got='{m['actual_in_suggestion']}'")
+    results.append(result)
+
+    return results
+
+# ═══════════════════════════════════════════════════════════════════
+# CATEGORY 11 — Genuine stress tests / edge cases
+# ═══════════════════════════════════════════════════════════════════
+def run_cat11():
+    log("=== CATEGORY 11: Edge case discovery (stress tests) ===")
+    results = []
+    tests = [
+        # Pathological inputs
+        {"id": "C11-01", "input": "", "desc": "empty_string"},
+        {"id": "C11-02", "input": " ", "desc": "whitespace_only"},
+        {"id": "C11-03", "input": "أ", "desc": "single_char"},
+        {"id": "C11-04", "input": "مستشفياتهم", "desc": "long_single_word"},
+        {"id": "C11-05", "input": "ذهبالولدالىالمدرسةوقابلالمعلمة", "desc": "no_spaces"},
+        {"id": "C11-06", "input": "...!؟،،؛؛::...", "desc": "all_punctuation"},
+        {"id": "C11-07", "input": "(([{هذا النص}]))", "desc": "unbalanced_brackets"},
+        {"id": "C11-08", "input": "\"هذا\" 'نص' «اختبار»", "desc": "mixed_quotes"},
+        # Boundary lengths (299, 300, 301 chars)
+        {"id": "C11-09", "input": ("يستخدم الذكاء الاصطناعي تقنيات التعلم العميق " * 10)[:299], "desc": "len_299"},
+        {"id": "C11-10", "input": ("يستخدم الذكاء الاصطناعي تقنيات التعلم العميق " * 10)[:300], "desc": "len_300"},
+        {"id": "C11-11", "input": ("يستخدم الذكاء الاصطناعي تقنيات التعلم العميق " * 10)[:301], "desc": "len_301"},
+        # Max disagreement: word that is both plausible spelling error AND grammatically ambiguous
+        {"id": "C11-12", "input": "يلعب الطلاب في الحديقه بعد المدرسه وقبل العشاء", "desc": "multi_stage_disagreement"},
+        # Correction identical to original (model returns same text)
+        {"id": "C11-13", "input": "الحمد لله", "desc": "model_returns_identical"},
+        # Very long repetitive text
+        {"id": "C11-14", "input": "مرحبا " * 100, "desc": "100x_repeated_word"},
+        # Mixed Arabic and English heavily
+        {"id": "C11-15", "input": "I went to the مدرسة and met the معلم in the فصل", "desc": "heavy_code_switch"},
+        # Dialectal variations
+        {"id": "C11-16", "input": "ايش هالحكي يا زلمة", "desc": "levantine_dialect"},
+        {"id": "C11-17", "input": "شنو تسوي هسه", "desc": "iraqi_dialect"},
+    ]
+    for test in tests:
+        log(f"  {test['id']}: '{test['input'][:40]}...' [{test['desc']}]")
+        # Track B only for stress tests (we want to see if pipeline crashes)
+        b = track_b_analyze(test['input'])
+        crashed = "error" in b and "suggestions" not in b
+        result = {
+            "id": test['id'], "category": 11, "input": test['input'][:200],
+            "desc": test['desc'], "input_len": len(test['input']),
+            "crashed": crashed,
+            "b_corrected": b.get('corrected', '')[:200] if not crashed else "CRASH",
+            "b_suggestions": len(b.get('suggestions', [])),
+            "error": b.get('error', None),
+        }
+        status = "💥 CRASH" if crashed else f"✓ ({len(b.get('suggestions',[]))} sugg)"
+        log(f"    {status}")
+        results.append(result)
+
+    # Race condition: 2 parallel requests with same input
+    log("\n  Race condition test (2 parallel requests):")
+    race_input = "كانت الفتيات يلعبون في الحديقه"
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as ex:
+        f1 = ex.submit(track_b_analyze, race_input)
+        f2 = ex.submit(track_b_analyze, race_input)
+        r1, r2 = f1.result(), f2.result()
+    race_match = r1.get('corrected') == r2.get('corrected') and len(r1.get('suggestions',[])) == len(r2.get('suggestions',[]))
+    race_result = {
+        "id": "C11-RACE", "category": 11, "input": race_input,
+        "desc": "parallel_race_condition",
+        "r1_corrected": r1.get('corrected', ''),
+        "r2_corrected": r2.get('corrected', ''),
+        "r1_suggestions": len(r1.get('suggestions', [])),
+        "r2_suggestions": len(r2.get('suggestions', [])),
+        "identical": race_match,
+    }
+    log(f"    Race test: identical={race_match}")
+    results.append(race_result)
+
+    return results
+
+# ═══════════════════════════════════════════════════════════════════
+# MAIN
+# ═══════════════════════════════════════════════════════════════════
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--stage', choices=['cat1', 'cat7', 'cat8x', 'cat9x', 'cat10x', 'cat11', 'all'], default='all')
+    args = parser.parse_args()
+
+    all_results = {"timestamp": datetime.now(timezone.utc).isoformat(), "api_base": API_BASE}
+
+    # Health check
+    log(f"Health check: {API_BASE}")
+    try:
+        resp = requests.get(f"{API_BASE}/api/health", timeout=10)
+        log(f"  OK: {resp.status_code}")
+        all_results['health'] = resp.json()
+    except Exception as e:
+        log(f"  FAIL: {e}")
+        return
+
+    if args.stage in ('cat1', 'all'):
+        all_results['cat1'] = run_cat1()
+    if args.stage in ('cat7', 'all'):
+        all_results['cat7'] = run_cat7()
+    if args.stage in ('cat8x', 'all'):
+        all_results['cat8x'] = run_cat8_expanded()
+    if args.stage in ('cat9x', 'all'):
+        all_results['cat9x'] = run_cat9_expanded()
+    if args.stage in ('cat10x', 'all'):
+        all_results['cat10x'] = run_cat10_expanded()
+    if args.stage in ('cat11', 'all'):
+        all_results['cat11'] = run_cat11()
+
+    output_path = os.path.join(os.path.dirname(__file__), 'deep_dive_expanded.json')
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(all_results, f, ensure_ascii=False, indent=2)
+    log(f"\nSaved to {output_path}")
+
+if __name__ == '__main__':
+    main()

tests/deep_dive_gaps.json

@@ -0,0 +1,260 @@
+{
+  "timestamp": "2026-06-20T19:33:59.110768+00:00",
+  "gap1_drift": {
+    "word_count": 187,
+    "char_count": 1104,
+    "total_suggestions": 0,
+    "front_half": 0,
+    "back_half": 0,
+    "coordinate_mismatches": [],
+    "a_spelling_changed": true,
+    "a_grammar_changed": true,
+    "a_punc_changed": true,
+    "suggestions": []
+  },
+  "gap2_priority": [
+    {
+      "id": "G2-01",
+      "input": "الطلاب اجتهدو في الامتحان",
+      "desc": "اجتهدو — spelling should add ا, grammar may do different fix. Grammar wins (priority 3 > 1)",
+      "a_spelling": "الطلاب اجتهدو في الامتحان",
+      "a_grammar": "الطلاب اجتهدو في الامتحان",
+      "a_punctuation": "الطلاب اجتهدو في الامتحان.",
+      "b_corrected": "الطلاب اجتهد في الامتحين",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "الامتحين",
+          "end": 25,
+          "id": "502647e7-18fd-41d2-b1c7-2978a3ee7704",
+          "locked": true,
+          "original": "الامتحان",
+          "priority": 3,
+          "start": 17,
+          "type": "grammar"
+        },
+        {
+          "alternatives": [
+            "اجتهد",
+            "اجتهدو"
+          ],
+          "confidence": 0.9,
+          "correction": "اجتهد",
+          "end": 13,
+          "id": "0a93f071-af36-4219-b6f5-d11e748c4601",
+          "locked": true,
+          "original": "اجتهدو",
+          "priority": 1,
+          "start": 7,
+          "type": "spelling"
+        }
+      ]
+    },
+    {
+      "id": "G2-02",
+      "input": "البنات ذهبو الى البيت",
+      "desc": "ذهبو — spelling could give ذهبوا, grammar could give ذهبن (fem). Grammar wins.",
+      "a_spelling": "البنات ذهبو إلى البيت",
+      "a_grammar": "البنات ذهبن الى البيت",
+      "a_punctuation": "البنات ذهبو الى البيت.",
+      "b_corrected": "البنات ذهبن إلى البيت.",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "ذهبن",
+          "end": 11,
+          "id": "0f00a9ab-1166-4e4d-8dd7-ae6dba1f9f1e",
+          "locked": true,
+          "original": "ذهبو",
+          "priority": 3,
+          "start": 7,
+          "type": "grammar"
+        },
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "البيت.",
+          "end": 21,
+          "id": "af3a0a21-5e1e-45f5-a1ad-9c3730b4ab25",
+          "locked": true,
+          "original": "البيت",
+          "priority": 2,
+          "start": 16,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [
+            "إلى",
+            "ال",
+            "الم",
+            "الى"
+          ],
+          "confidence": 0.9,
+          "correction": "إلى",
+          "end": 15,
+          "id": "1b7096dc-6043-4e1a-9de3-d59204327b86",
+          "locked": true,
+          "original": "الى",
+          "priority": 1,
+          "start": 12,
+          "type": "spelling"
+        }
+      ]
+    },
+    {
+      "id": "G2-03",
+      "input": "وفجأه سقطت الكتب",
+      "desc": "وفجأه — spelling may fix ه→ة; punctuation may want comma after it. Overlap?",
+      "a_spelling": "وفجأه سقطت الكتب",
+      "a_grammar": "وفجأة سقطت الكتب",
+      "a_punctuation": "وفجأه سقطت الكتب.",
+      "b_corrected": "وفجأة سقطت الكتب.",
+      "b_suggestions": [
+        {
+          "alternatives": [],
+          "confidence": 0.8,
+          "correction": "الكتب.",
+          "end": 16,
+          "id": "fc257e46-4368-4d32-acb0-de5b6d461aaf",
+          "locked": true,
+          "original": "الكتب",
+          "priority": 2,
+          "start": 11,
+          "type": "punctuation"
+        },
+        {
+          "alternatives": [],
+          "confidence": 1.0,
+          "correction": "وفجأة",
+          "end": 5,
+          "id": "7397e7e6-e238-4ed4-a184-461f576a74f6",
+          "locked": true,
+          "original": "وفجأه",
+          "priority": 1,
+          "start": 0,
+          "type": "spelling"
+        }
+      ]
+    }
+  ],
+  "gap3_dropped": {
+    "tests": [
+      {
+        "input": "الطلاب الذين اجتهدو في دراستهم حققو نتائج ممتازه في الامتحانات",
+        "a_spell_diffs": [
+          {
+            "word_idx": 2,
+            "original": "اجتهدو",
+            "corrected": "اجتهد"
+          },
+          {
+            "word_idx": 5,
+            "original": "حققو",
+            "corrected": "حقوق"
+          }
+        ],
+        "a_gram_diffs": [
+          {
+            "word_idx": 5,
+            "original": "حققو",
+            "corrected": "حققوا"
+          },
+          {
+            "word_idx": 7,
+            "original": "ممتازه",
+            "corrected": "ممتازة"
+          }
+        ],
+        "a_punc_diffs": [
+          {
+            "word_idx": 9,
+            "original": "الامتحانات",
+            "corrected": "الامتحانات."
+          }
+        ],
+        "b_suggestion_count": 4,
+        "dropped_spell": [],
+        "dropped_gram": [],
+        "dropped_punc": []
+      }
+    ]
+  },
+  "gap4_rare": {
+    "tests": [
+      {
+        "id": "R-01",
+        "input": "استوقفني المشهد فتأملته مليا",
+        "domain": "literary",
+        "output": "استوقفني المشهد فتأملتة مليا",
+        "changed": true
+      },
+      {
+        "id": "R-02",
+        "input": "تستأثر القوى العظمى بالنفوذ الدولي",
+        "domain": "political_literary",
+        "output": "تستأثر القوى العظمى بالنفوذ الدولي",
+        "changed": false
+      },
+      {
+        "id": "R-03",
+        "input": "استقطب المؤتمر ثلة من العلماء الأفذاذ",
+        "domain": "formal_rare",
+        "output": "استقطب المؤتمر ثلة من العلماء الأفذاذ",
+        "changed": false
+      },
+      {
+        "id": "R-04",
+        "input": "يتسنى للمرء أن يستشف الحقيقة من بين السطور",
+        "domain": "literary_verb",
+        "output": "يتسنى للمرء أن يكتشف الحقيقة من بين السطور",
+        "changed": true
+      },
+      {
+        "id": "R-05",
+        "input": "ألقى المحاضر خطبة عصماء استحوذت على إعجاب الحاضرين",
+        "domain": "oratory",
+        "output": "ألقى المحاضر خطبة علماء استحوذت على إعجاب الحاضرين",
+        "changed": true
+      },
+      {
+        "id": "R-06",
+        "input": "تمخض الاجتماع عن قرارات مصيرية",
+        "domain": "formal_verb",
+        "output": "تمخض الاجتماع عن قرارات مصيرية",
+        "changed": false
+      },
+      {
+        "id": "R-07",
+        "input": "أرهقته المسغبة فاستكان للقدر",
+        "domain": "classical",
+        "output": "طريقتة المسببة فاستكان القدر",
+        "changed": true
+      },
+      {
+        "id": "R-08",
+        "input": "نستشرف آفاق المستقبل بثقة واقتدار",
+        "domain": "formal_speech",
+        "output": "نستشرف آفاق المستقبل بثقة واقتدار",
+        "changed": false
+      },
+      {
+        "id": "R-09",
+        "input": "اعتراه القلق فتملكه الأرق",
+        "domain": "literary_psych",
+        "output": "اعتراه القلق فتملكة الأرق",
+        "changed": true
+      },
+      {
+        "id": "R-10",
+        "input": "استأنف العمل بعد فترة من التقاعس",
+        "domain": "formal_verb",
+        "output": "استأنف العمل بعد فترة من التقاعد",
+        "changed": true
+      }
+    ],
+    "fp_count": 6,
+    "total": 10
+  }
+}

tests/deep_dive_gaps.py

@@ -0,0 +1,295 @@
+"""
+Gap-filler tests for items explicitly requested in the prompt but not yet covered:
+1. 200+ word cumulative drift test (Cat 10)
+2. Lower-priority-wins limitation (Cat 4)
+3. Systematic dropped patch logging (Cat 3)
+4. Rare/literary vocabulary overcorrection (Cat 2)
+"""
+import sys, os, json, time, requests
+from datetime import datetime, timezone
+
+API_BASE = "https://bayan10-bayan-api.hf.space"
+TIMEOUT = 60
+
+def api_call(endpoint, text, retries=2):
+    url = f"{API_BASE}{endpoint}"
+    for attempt in range(retries + 1):
+        try:
+            t0 = time.time()
+            resp = requests.post(url, json={"text": text}, timeout=TIMEOUT)
+            elapsed = int((time.time() - t0) * 1000)
+            if resp.status_code == 200:
+                data = resp.json()
+                data['_elapsed_ms'] = elapsed
+                return data
+            else:
+                if attempt < retries:
+                    time.sleep(2)
+                    continue
+                return {"error": f"HTTP {resp.status_code}: {resp.text[:200]}", "_elapsed_ms": elapsed}
+        except Exception as e:
+            return {"error": str(e)}
+
+def log(msg):
+    print(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}", flush=True)
+
+results = {"timestamp": datetime.now(timezone.utc).isoformat()}
+
+# ═══════════════════════════════════════════════════════════════
+# GAP 1: 200+ word cumulative drift test (Cat 10)
+# ═══════════════════════════════════════════════════════════════
+log("=== GAP 1: 200+ word cumulative drift test ===")
+
+# Build a 200+ word paragraph with deliberate errors throughout
+long_para = (
+    "كانت الفتيات يلعبون في الحديقه الجميله وفجأه سقطت احداهن وبدءت تبكي بشده "
+    "ذهب الولد الى المدرسه وقابل المعلمه واخذ الكتاب وبدأ يقرأ بتركيز شديد "
+    "ان الذكاء الاصطناعي يلعب دورا هاما في تطوير التكنولوجيا الحديثه ولذالك يجب الاهتمام بة "
+    "هذة المدينه جميله جدا ومناخها معتدل طوال العام وسكانها طيبون ومحبون للخير "
+    "الطلاب الذين اجتهدو في دراستهم حققو نتائج ممتازه في الامتحانات النهائيه "
+    "سافر محمد إلى دبي للعمل في شركة جوجل وقابل أصدقاءه القدامى هناك "
+    "يستخدم الذكاء الاصطناعي تقنيات التعلم العميق في معالجة البيانات الضخمة والتحليل "
+    "القاهرة عاصمة جمهورية مصر العربية وأكبر مدنها وتقع على ضفاف نهر النيل العظيم "
+    "تتراوح درجات الحرارة بين خمس وعشرين وثلاثين درجة مئوية في فصل الصيف الحار "
+    "الحمد لله رب العالمين الرحمن الرحيم مالك يوم الدين إياك نعبد وإياك نستعين "
+    "بسم الله الرحمن الرحيم نبدأ هذة المحاضره عن اهمية التعليم في حياة الانسان "
+    "يحب الأطفال اللعب في الحديقة وركوب الدراجات والجري بين الأشجار والزهور الجميلة "
+    "إن العلم نور والجهل ظلام فاحرصوا على طلب العلم من المهد إلى اللحد "
+    "كان الرجل يمشي في الشارع وفجأه رأى صديقه القديم فسلم عليه وتحدثا طويلا "
+    "المعلم الذي يحب عمله يجتهد في تعليم طلابه ويحرص على نجاحهم في الحياه "
+)
+
+word_count = len(long_para.split())
+char_count = len(long_para)
+log(f"  Input: {word_count} words, {char_count} chars")
+
+# Track A: each model on the full long text
+log("  Running Track A (each model independently on original)...")
+a_spell = api_call("/api/spelling", long_para)
+a_gram = api_call("/api/grammar", long_para)
+a_punc = api_call("/api/punctuation", long_para)
+
+# Track B: full pipeline
+log("  Running Track B (full pipeline)...")
+b = api_call("/api/analyze", long_para)
+
+sugg = b.get("suggestions", [])
+mid_char = char_count // 2
+
+# Verify ALL coordinates
+coord_mismatches = []
+for s in sugg:
+    start, end = s.get('start', 0), s.get('end', 0)
+    expected_text = long_para[start:end]
+    actual_text = s.get('original', '')
+    if expected_text != actual_text:
+        coord_mismatches.append({
+            "start": start, "end": end,
+            "expected": expected_text,
+            "actual": actual_text,
+            "correction": s.get('correction', ''),
+            "type": s.get('type', ''),
+        })
+
+back_half = [s for s in sugg if s.get('start', 0) >= mid_char]
+front_half = [s for s in sugg if s.get('start', 0) < mid_char]
+
+# Log every suggestion with its verified coordinate
+log(f"  Total: {len(sugg)} suggestions, {len(coord_mismatches)} coordinate mismatches")
+log(f"  Front half ({mid_char} chars): {len(front_half)} suggestions")
+log(f"  Back half: {len(back_half)} suggestions")
+for s in sugg:
+    st, en = s.get('start',0), s.get('end',0)
+    in_back = "BACK" if st >= mid_char else "FRONT"
+    verified = "✓" if long_para[st:en] == s.get('original','') else "✗ MISMATCH"
+    log(f"    [{in_back}] [{st}:{en}] '{s.get('original','')}' → '{s.get('correction','')}' ({s.get('type','')}) {verified}")
+
+for m in coord_mismatches:
+    log(f"    MISMATCH: [{m['start']}:{m['end']}] expected='{m['expected']}' actual='{m['actual']}'")
+
+results['gap1_drift'] = {
+    "word_count": word_count, "char_count": char_count,
+    "total_suggestions": len(sugg),
+    "front_half": len(front_half), "back_half": len(back_half),
+    "coordinate_mismatches": coord_mismatches,
+    "a_spelling_changed": a_spell.get("corrected_text","") != long_para,
+    "a_grammar_changed": a_gram.get("corrected_text","") != long_para,
+    "a_punc_changed": a_punc.get("corrected_text","") != long_para,
+    "suggestions": sugg,
+}
+
+# ═══════════════════════════════════════════════════════════════
+# GAP 2: Lower-priority-wins limitation doc (Cat 4)
+# ═══════════════════════════════════════════════════════════════
+log("\n=== GAP 2: Lower-priority stage was more important (Cat 4) ===")
+
+# Construct case: spelling corrects اجتهدو→اجتهدوا (correct, priority 1)
+# but grammar might also touch it with a different correction (priority 3)
+# Grammar WINS because higher priority. But what if grammar is wrong here?
+gap2_tests = [
+    {
+        "id": "G2-01",
+        "input": "الطلاب اجتهدو في الامتحان",
+        "desc": "اجتهدو — spelling should add ا, grammar may do different fix. Grammar wins (priority 3 > 1)",
+    },
+    {
+        "id": "G2-02",
+        "input": "البنات ذهبو الى البيت",
+        "desc": "ذهبو — spelling could give ذهبوا, grammar could give ذهبن (fem). Grammar wins.",
+    },
+    {
+        "id": "G2-03",
+        "input": "وفجأه سقطت الكتب",
+        "desc": "وفجأه — spelling may fix ه→ة; punctuation may want comma after it. Overlap?",
+    },
+]
+
+for test in gap2_tests:
+    log(f"  {test['id']}: {test['input']}")
+    a_sp = api_call("/api/spelling", test['input'])
+    a_gr = api_call("/api/grammar", test['input'])
+    a_pu = api_call("/api/punctuation", test['input'])
+    b = api_call("/api/analyze", test['input'])
+
+    a_sp_out = a_sp.get("corrected_text", test['input'])
+    a_gr_out = a_gr.get("corrected_text", test['input'])
+    a_pu_out = a_pu.get("corrected_text", test['input'])
+
+    log(f"    A_spell: {a_sp_out}")
+    log(f"    A_gram:  {a_gr_out}")
+    log(f"    A_punc:  {a_pu_out}")
+    log(f"    B_final: {b.get('corrected','')}")
+    log(f"    B_sugg:  {len(b.get('suggestions',[]))}")
+
+    # Which stage's correction won for each word?
+    b_sugg = b.get('suggestions', [])
+    for s in b_sugg:
+        log(f"      [{s.get('type','')}] [{s.get('start',0)}:{s.get('end',0)}] '{s.get('original','')}' → '{s.get('correction','')}'")
+
+    test['a_spelling'] = a_sp_out
+    test['a_grammar'] = a_gr_out
+    test['a_punctuation'] = a_pu_out
+    test['b_corrected'] = b.get('corrected', '')
+    test['b_suggestions'] = b_sugg
+
+results['gap2_priority'] = gap2_tests
+
+# ═══════════════════════════════════════════════════════════════
+# GAP 3: Systematic dropped patch logging (Cat 3)
+# ═══════════════════════════════════════════════════════════════
+log("\n=== GAP 3: Systematic dropped patch comparison (Cat 3) ===")
+
+# For each test: run all 3 models independently, count expected patches,
+# compare with actual Track B patches. Any patch Track A produces but
+# Track B doesn't = dropped patch.
+gap3_tests = [
+    "كانت الفتيات يلعبون في الحديقه وفجأه سقطت احداهن وبدءت تبكي بشده",
+    "ان الذكاء الاصطناعي يلعب دورا هاما ولذالك يجب الاهتمام بة",
+    "هذة المدينه جميله جدا ومناخها معتدل طوال العام",
+    "ذهب الولد الى المكتبه وقرا كتاب مفيد",
+    "الطلاب الذين اجتهدو في دراستهم حققو نتائج ممتازه في الامتحانات",
+]
+
+for i, text in enumerate(gap3_tests):
+    log(f"  Test {i+1}: {text[:50]}...")
+    a_sp = api_call("/api/spelling", text)
+    a_gr = api_call("/api/grammar", text)
+    a_pu = api_call("/api/punctuation", text)
+    b = api_call("/api/analyze", text)
+
+    a_sp_out = a_sp.get("corrected_text", text)
+    a_gr_out = a_gr.get("corrected_text", text)
+    a_pu_out = a_pu.get("corrected_text", text)
+
+    # Find word-level changes from each model
+    def word_diffs(orig, corrected):
+        o_words = orig.split()
+        c_words = corrected.split()
+        diffs = []
+        for j, (ow, cw) in enumerate(zip(o_words, c_words)):
+            if ow != cw:
+                diffs.append({"word_idx": j, "original": ow, "corrected": cw})
+        return diffs
+
+    sp_diffs = word_diffs(text, a_sp_out)
+    gr_diffs = word_diffs(text, a_gr_out)
+    pu_diffs = word_diffs(text, a_pu_out)
+
+    b_sugg = b.get('suggestions', [])
+    b_corrections = set()
+    for s in b_sugg:
+        b_corrections.add(s.get('original', ''))
+
+    # Track A produced these corrections; check which survived to Track B
+    dropped_spell = [d for d in sp_diffs if d['original'] not in b_corrections and d['corrected'] != d['original']]
+    dropped_gram = [d for d in gr_diffs if d['original'] not in b_corrections and d['corrected'] != d['original']]
+    dropped_punc = [d for d in pu_diffs if d['original'] not in b_corrections and d['corrected'] != d['original']]
+
+    log(f"    Track A changes: spell={len(sp_diffs)}, gram={len(gr_diffs)}, punc={len(pu_diffs)}")
+    log(f"    Track B suggestions: {len(b_sugg)}")
+    log(f"    Dropped: spell={len(dropped_spell)}, gram={len(dropped_gram)}, punc={len(dropped_punc)}")
+
+    for d in dropped_spell:
+        log(f"      DROPPED SPELL: '{d['original']}' → '{d['corrected']}' (reason: likely filter blocked)")
+    for d in dropped_gram:
+        log(f"      DROPPED GRAM: '{d['original']}' → '{d['corrected']}' (reason: likely StageLocker)")
+    for d in dropped_punc:
+        log(f"      DROPPED PUNC: '{d['original']}' → '{d['corrected']}' (reason: likely lock/cap/safety)")
+
+results[f'gap3_dropped'] = {
+    "tests": [
+        {
+            "input": text,
+            "a_spell_diffs": word_diffs(text, api_call("/api/spelling", text).get("corrected_text", text)) if False else sp_diffs,
+            "a_gram_diffs": gr_diffs,
+            "a_punc_diffs": pu_diffs,
+            "b_suggestion_count": len(b_sugg),
+            "dropped_spell": dropped_spell,
+            "dropped_gram": dropped_gram,
+            "dropped_punc": dropped_punc,
+        }
+        for text, sp_diffs, gr_diffs, pu_diffs, b_sugg in [(text, sp_diffs, gr_diffs, pu_diffs, b_sugg)]
+    ]
+}
+
+# ═══════════════════════════════════════════════════════════════
+# GAP 4: Rare/literary vocabulary (Cat 2)
+# ═══════════════════════════════════════════════════════════════
+log("\n=== GAP 4: Rare/literary vocabulary overcorrection (Cat 2) ===")
+
+rare_tests = [
+    {"id": "R-01", "input": "استوقفني المشهد فتأملته مليا", "domain": "literary"},
+    {"id": "R-02", "input": "تستأثر القوى العظمى بالنفوذ الدولي", "domain": "political_literary"},
+    {"id": "R-03", "input": "استقطب المؤتمر ثلة من العلماء الأفذاذ", "domain": "formal_rare"},
+    {"id": "R-04", "input": "يتسنى للمرء أن يستشف الحقيقة من بين السطور", "domain": "literary_verb"},
+    {"id": "R-05", "input": "ألقى المحاضر خطبة عصماء استحوذت على إعجاب الحاضرين", "domain": "oratory"},
+    {"id": "R-06", "input": "تمخض الاجتماع عن قرارات مصيرية", "domain": "formal_verb"},
+    {"id": "R-07", "input": "أرهقته المسغبة فاستكان للقدر", "domain": "classical"},
+    {"id": "R-08", "input": "نستشرف آفاق المستقبل بثقة واقتدار", "domain": "formal_speech"},
+    {"id": "R-09", "input": "اعتراه القلق فتملكه الأرق", "domain": "literary_psych"},
+    {"id": "R-10", "input": "استأنف العمل بعد فترة من التقاعس", "domain": "formal_verb"},
+]
+
+fp_count = 0
+for test in rare_tests:
+    a = api_call("/api/spelling", test['input'])
+    a_out = a.get("corrected_text", test['input'])
+    changed = a_out != test['input']
+    if changed:
+        fp_count += 1
+        log(f"  ⚠ {test['id']}: '{test['input'][:40]}...' → '{a_out[:40]}...' [{test['domain']}]")
+    else:
+        log(f"  ✓ {test['id']}: no change [{test['domain']}]")
+    test['output'] = a_out
+    test['changed'] = changed
+
+log(f"  Rare/literary FP rate: {fp_count}/{len(rare_tests)} ({fp_count*100//len(rare_tests)}%)")
+results['gap4_rare'] = {"tests": rare_tests, "fp_count": fp_count, "total": len(rare_tests)}
+
+# ═══════════════════════════════════════════════════════════════
+# SAVE
+# ═══════════════════════════════════════════════════════════════
+output_path = os.path.join(os.path.dirname(__file__), 'deep_dive_gaps.json')
+with open(output_path, 'w', encoding='utf-8') as f:
+    json.dump(results, f, ensure_ascii=False, indent=2)
+log(f"\nSaved to {output_path}")

tests/deep_dive_output.json

@@ -0,0 +1,671 @@
+{
+  "timestamp": "2026-06-20T19:00:06.993902+00:00",
+  "api_base": "https://bayan10-bayan-api.hf.space",
+  "health": {
+    "environment": "huggingface_spaces",
+    "mode": "hf_spaces_local",
+    "models": {
+      "autocomplete": true,
+      "grammar": true,
+      "punctuation": true,
+      "spelling": true,
+      "summarization": true
+    },
+    "note": "Free tier: summarization local, other models return input unchanged",
+    "status": "healthy",
+    "supabase": {
+      "configured": true
+    }
+  },
+  "pipeline_tests": [
+    {
+      "id": "C3-01",
+      "category": 3,
+      "input": "كانت الفتيات يلعبون في الحديقه وفجأه سقطت احداهن وبدءت تبكي بشده",
+      "track_a": {
+        "spelling": "كانت الفتيات يلعبون في الحديقه وفجأه سقطت احداهن وبدءت تبكي بشدة",
+        "spelling_changed": true,
+        "grammar": "كانت الفتيات يلعبن في الحديقة وفجأة سقطت إحدىهن وبدأت تبكي بشدة",
+        "grammar_changed": true,
+        "punctuation": "كانت الفتيات يلعبون في الحديقه وفجأه، سقطت احداهن وبدءت تبكي بشده",
+        "punctuation_changed": true
+      },
+      "track_b": {
+        "corrected": "كانت الفتيات يلعبن في الحديقة وفجأة سقطت إحدىهن وبدأت تبكي بشدة.",
+        "suggestions": [
+          {
+            "alternatives": [],
+            "confidence": 1.0,
+            "correction": "يلعبن",
+            "end": 19,
+            "id": "e984c773-8d33-4a30-b5b8-49cee91e1095",
+            "locked": true,
+            "original": "يلعبون",
+            "priority": 3,
+            "start": 13,
+            "type": "grammar"
+          },
+          {
+            "alternatives": [],
+            "confidence": 1.0,
+            "correction": "إحدىهن وبدأت",
+            "end": 54,
+            "id": "38054ed7-9bd2-4e04-9314-b4a63b84ad07",
+            "locked": true,
+            "original": "احداهن وبدءت",
+            "priority": 3,
+            "start": 42,
+            "type": "grammar"
+          },
+          {
+            "alternatives": [],
+            "confidence": 0.8,
+            "correction": "بشدة.",
+            "end": 64,
+            "id": "16e72e95-6326-4365-a0f3-ad2602bcfc49",
+            "locked": true,
+            "original": "بشده",
+            "priority": 2,
+            "start": 60,
+            "type": "punctuation"
+          },
+          {
+            "alternatives": [],
+            "confidence": 1.0,
+            "correction": "الحديقة وفجأة",
+            "end": 36,
+            "id": "1de0b7c2-e2e5-45e3-8ba3-6fe062ee8fcc",
+            "locked": true,
+            "original": "الحديقه وفجأه",
+            "priority": 1,
+            "start": 23,
+            "type": "spelling"
+          }
+        ],
+        "timing_ms": {
+          "grammar_ms": 4561,
+          "punctuation_ms": 1492,
+          "spelling_ms": 1529,
+          "total_ms": 7587
+        }
+      }
+    },
+    {
+      "id": "C3-02",
+      "category": 3,
+      "input": "ان الذكاء الاصطناعي يلعب دورا هاما ولذالك يجب الاهتمام بة",
+      "track_a": {
+        "spelling": "ان الذكاء الاصطناعي يلعب دورا هاما ولذالك يجب الاهتمام بة",
+        "spelling_changed": false,
+        "grammar": "ان الذكاء الاصطناعي يلعب دورا هاما ولذلك يجب الاهتمام به",
+        "grammar_changed": true,
+        "punctuation": "ان الذكاء الاصطناعي يلعب دورا هاما ولذالك؛ يجب الاهتمام بة",
+        "punctuation_changed": true
+      },
+      "track_b": {
+        "corrected": "ان الذكاء الاصطناعي يلعب دورا هاما ولذلك يجب الاهتمام به",
+        "suggestions": [
+          {
+            "alternatives": [],
+            "confidence": 1.0,
+            "correction": "ولذلك",
+            "end": 41,
+            "id": "9870eb8d-0bf7-4a58-90cb-940b5475a37e",
+            "locked": true,
+            "original": "ولذالك",
+            "priority": 3,
+            "start": 35,
+            "type": "grammar"
+          },
+          {
+            "alternatives": [],
+            "confidence": 1.0,
+            "correction": "به",
+            "end": 57,
+            "id": "ea9f3fca-eee1-4597-8f4a-00f50558d510",
+            "locked": true,
+            "original": "بة",
+            "priority": 1,
+            "start": 55,
+            "type": "spelling"
+          }
+        ],
+        "timing_ms": {
+          "grammar_ms": 1304,
+          "punctuation_ms": 1050,
+          "spelling_ms": 1193,
+          "total_ms": 3549
+        }
+      }
+    },
+    {
+      "id": "C3-03",
+      "category": 3,
+      "input": "التزم الر��اضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة في بناء كتلة عضلية قوية ويا له من التزام حديدي يثير الإعجاب",
+      "track_a": {
+        "spelling": "التزم الرياضي بتناول وجبات الصحية وحساب سعادتة بدقة رغبة في بناء كتلة عملية قوية ويا له من التزام حديدي يثير الإعجاب",
+        "spelling_changed": true,
+        "grammar": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة في بناء كتلة عضلية قوية ويا له من التزام حديدي يثير الإعجاب",
+        "grammar_changed": false,
+        "punctuation": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة؛ في بناء كتلة عضلية قوية ويا له، من التزام حديدي يثير الإعجاب",
+        "punctuation_changed": true
+      },
+      "track_b": {
+        "corrected": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة؛ في بناء كتلة عضلية قوية ويا له، من التزام حديدي يثير الإعجاب",
+        "suggestions": [
+          {
+            "alternatives": [],
+            "confidence": 0.8,
+            "correction": "رغبة؛",
+            "end": 57,
+            "id": "b7e29bf0-2565-4b46-b815-58e1b56717c1",
+            "locked": true,
+            "original": "رغبة",
+            "priority": 2,
+            "start": 53,
+            "type": "punctuation"
+          },
+          {
+            "alternatives": [],
+            "confidence": 0.8,
+            "correction": "له،",
+            "end": 88,
+            "id": "6d1e2b65-d2a4-41f1-a803-ce06e93e79c4",
+            "locked": true,
+            "original": "له",
+            "priority": 2,
+            "start": 86,
+            "type": "punctuation"
+          }
+        ],
+        "timing_ms": {
+          "grammar_ms": 6012,
+          "punctuation_ms": 2080,
+          "spelling_ms": 2197,
+          "total_ms": 10291
+        }
+      }
+    },
+    {
+      "id": "C3-04",
+      "category": 3,
+      "input": "هذة المدينه جميله جدا ومناخها معتدل طوال العام",
+      "track_a": {
+        "spelling": "هذة المدينه جميله جدا ومناخها معتدل طوال العام",
+        "spelling_changed": false,
+        "grammar": "هذه المدينة جميلة جدا ومناخها معتدل طوال العام",
+        "grammar_changed": true,
+        "punctuation": "هذة المدينه جميله جدا ومناخها معتدل طوال العام.",
+        "punctuation_changed": true
+      },
+      "track_b": {
+        "corrected": "هذه المدينة جميلة جدا ومناخها معتدل طوال العام.",
+        "suggestions": [
+          {
+            "alternatives": [],
+            "confidence": 0.8,
+            "correction": "العام.",
+            "end": 46,
+            "id": "a4de368f-4ae7-451a-bbe2-ff7fca6b3f3b",
+            "locked": true,
+            "original": "العام",
+            "priority": 2,
+            "start": 41,
+            "type": "punctuation"
+          },
+          {
+            "alternatives": [],
+            "confidence": 1.0,
+            "correction": "هذه المدينة جميلة",
+            "end": 17,
+            "id": "9ff77094-1e33-4946-a343-317f51b8b539",
+            "locked": true,
+            "original": "هذة المدينه جميله",
+            "priority": 1,
+            "start": 0,
+            "type": "spelling"
+          }
+        ],
+        "timing_ms": {
+          "grammar_ms": 1461,
+          "punctuation_ms": 804,
+          "spelling_ms": 970,
+          "total_ms": 3236
+        }
+      }
+    },
+    {
+      "id": "C3-05",
+      "category": 3,
+      "input": "الطلاب الذين اجتهدو في دراستهم حققو نتائج ممتازه في الامتحانات",
+      "track_a": {
+        "spelling": "الطلاب الذين اجتهد في دراستهم حقوق نتائج ممتازه في الامتحانات",
+        "spelling_changed": true,
+        "grammar": "الطلاب الذين اجتهدو في دراستهم حققو نتائج ممتازه في الامتحانات",
+        "grammar_changed": false,
+        "punctuation": "الطلاب الذين اجتهدو في دراستهم حققو نتائج ممتازه في الامتحانات",
+        "punctuation_changed": false
+      },
+      "track_b": {
+        "corrected": "",
+        "suggestions": [],
+        "timing_ms": {}
+      }
+    },
+    {
+      "id": "C4-01",
+      "category": 4,
+      "input": "كانت الفتيات يلعبون في الحديقه",
+      "runs": [
+        {
+          "run": 1,
+          "corrected": "",
+          "suggestions": []
+        },
+        {
+          "run": 2,
+          "corrected": "",
+          "suggestions": []
+        },
+        {
+          "run": 3,
+          "corrected": "",
+          "suggestions": []
+        }
+      ],
+      "deterministic": true
+    },
+    {
+      "id": "C4-02",
+      "category": 4,
+      "input": "ذهب الى المدرسه وقابل المعلمه",
+      "runs": [
+        {
+          "run": 1,
+          "corrected": "ذهب إلى المدرسة وقابل المعلمة.",
+          "suggestions": [
+            {
+              "alternatives": [],
+              "confidence": 0.8,
+              "correction": "المعلمة.",
+              "end": 29,
+              "id": "3579ef5d-9295-46a9-8056-5a0b15dced2d",
+              "locked": true,
+              "original": "المعلمه",
+              "priority": 2,
+              "start": 22,
+              "type": "punctuation"
+            },
+            {
+              "alternatives": [],
+              "confidence": 1.0,
+              "correction": "المدرسة",
+              "end": 15,
+              "id": "af7b8dd8-f85a-4632-a7c9-b9b733d7e019",
+              "locked": true,
+              "original": "المدرسه",
+              "priority": 1,
+              "start": 8,
+              "type": "spelling"
+            },
+            {
+              "alternatives": [
+                "إلى",
+                "ال",
+                "الم",
+                "الى"
+              ],
+              "confidence": 0.9,
+              "correction": "إلى",
+              "end": 7,
+              "id": "cd3a78f0-afbc-42d0-8bba-c60ce884dfdf",
+              "locked": true,
+              "original": "الى",
+              "priority": 1,
+              "start": 4,
+              "type": "spelling"
+            }
+          ]
+        },
+        {
+          "run": 2,
+          "corrected": "ذهب إلى المدرسة وقابل المعلمة.",
+          "suggestions": [
+            {
+              "alternatives": [],
+              "confidence": 0.8,
+              "correction": "المعلمة.",
+              "end": 29,
+              "id": "4263a3c3-69cc-40a7-884f-a6e9bfd17eb1",
+              "locked": true,
+              "original": "المعلمه",
+              "priority": 2,
+              "start": 22,
+              "type": "punctuation"
+            },
+            {
+              "alternatives": [],
+              "confidence": 1.0,
+              "correction": "المدرسة",
+              "end": 15,
+              "id": "3c062f0a-95b6-4eee-bd80-36fc9b295206",
+              "locked": true,
+              "original": "المدرسه",
+              "priority": 1,
+              "start": 8,
+              "type": "spelling"
+            },
+            {
+              "alternatives": [
+                "إلى",
+                "ال",
+                "الم",
+                "الى"
+              ],
+              "confidence": 0.9,
+              "correction": "إلى",
+              "end": 7,
+              "id": "beb1ecbe-3278-47d5-bb14-d28f1eec5b47",
+              "locked": true,
+              "original": "الى",
+              "priority": 1,
+              "start": 4,
+              "type": "spelling"
+            }
+          ]
+        },
+        {
+          "run": 3,
+          "corrected": "ذهب إلى المدرسة وقابل المعلمة.",
+          "suggestions": [
+            {
+              "alternatives": [],
+              "confidence": 0.8,
+              "correction": "المعلمة.",
+              "end": 29,
+              "id": "5361ba1b-5c5f-4740-84be-1c4d96c665db",
+              "locked": true,
+              "original": "المعلمه",
+              "priority": 2,
+              "start": 22,
+              "type": "punctuation"
+            },
+            {
+              "alternatives": [],
+              "confidence": 1.0,
+              "correction": "المدرسة",
+              "end": 15,
+              "id": "f0450147-9d7a-4754-a4fe-403a07219c39",
+              "locked": true,
+              "original": "المدرسه",
+              "priority": 1,
+              "start": 8,
+              "type": "spelling"
+            },
+            {
+              "alternatives": [
+                "إلى",
+                "ال",
+                "الم",
+                "الى"
+              ],
+              "confidence": 0.9,
+              "correction": "إلى",
+              "end": 7,
+              "id": "a8278394-1555-4d01-ba94-1325efc0a97c",
+              "locked": true,
+              "original": "الى",
+              "priority": 1,
+              "start": 4,
+              "type": "spelling"
+            }
+          ]
+        }
+      ],
+      "deterministic": true
+    },
+    {
+      "id": "C4-03",
+      "category": 4,
+      "input": "ان الطالبات ذهبو الى الجامعه",
+      "runs": [
+        {
+          "run": 1,
+          "corrected": "إن الطالبات ذهبن ذه��وا الجامعة.",
+          "suggestions": [
+            {
+              "alternatives": [],
+              "confidence": 1.0,
+              "correction": "ذهبن",
+              "end": 16,
+              "id": "bc1d01e1-8d6b-4bda-bbe0-199e841d0f3d",
+              "locked": true,
+              "original": "ذهبو",
+              "priority": 3,
+              "start": 12,
+              "type": "grammar"
+            },
+            {
+              "alternatives": [],
+              "confidence": 0.8,
+              "correction": "الجامعة.",
+              "end": 28,
+              "id": "8cdb866c-0c6f-4cb1-a4ef-d00be9b455f7",
+              "locked": true,
+              "original": "الجامعه",
+              "priority": 2,
+              "start": 21,
+              "type": "punctuation"
+            },
+            {
+              "alternatives": [],
+              "confidence": 1.0,
+              "correction": "إن",
+              "end": 2,
+              "id": "027f98a7-668c-463f-9ecc-acaad6b959b2",
+              "locked": true,
+              "original": "ان",
+              "priority": 1,
+              "start": 0,
+              "type": "spelling"
+            },
+            {
+              "alternatives": [
+                "ذهبوا",
+                "ال",
+                "الم",
+                "الى"
+              ],
+              "confidence": 0.9,
+              "correction": "ذهبوا",
+              "end": 20,
+              "id": "8aee308b-6200-4c92-b6d1-95333a112ce0",
+              "locked": true,
+              "original": "الى",
+              "priority": 1,
+              "start": 17,
+              "type": "spelling"
+            }
+          ]
+        },
+        {
+          "run": 2,
+          "corrected": "إن الطالبات ذهبن ذهبوا الجامعة.",
+          "suggestions": [
+            {
+              "alternatives": [],
+              "confidence": 1.0,
+              "correction": "ذهبن",
+              "end": 16,
+              "id": "0c9ec931-ea50-423c-8429-89a100e1c226",
+              "locked": true,
+              "original": "ذهبو",
+              "priority": 3,
+              "start": 12,
+              "type": "grammar"
+            },
+            {
+              "alternatives": [],
+              "confidence": 0.8,
+              "correction": "الجامعة.",
+              "end": 28,
+              "id": "c67960b7-36f0-480a-8e85-716c57465107",
+              "locked": true,
+              "original": "الجامعه",
+              "priority": 2,
+              "start": 21,
+              "type": "punctuation"
+            },
+            {
+              "alternatives": [],
+              "confidence": 1.0,
+              "correction": "إن",
+              "end": 2,
+              "id": "787d7736-29aa-4625-90ad-e1248acb2d48",
+              "locked": true,
+              "original": "ان",
+              "priority": 1,
+              "start": 0,
+              "type": "spelling"
+            },
+            {
+              "alternatives": [
+                "ذهبوا",
+                "ال",
+                "الم",
+                "الى"
+              ],
+              "confidence": 0.9,
+              "correction": "ذهبوا",
+              "end": 20,
+              "id": "69c96488-d579-441c-89ea-3b66477f1f2d",
+              "locked": true,
+              "original": "الى",
+              "priority": 1,
+              "start": 17,
+              "type": "spelling"
+            }
+          ]
+        },
+        {
+          "run": 3,
+          "corrected": "إن الطالبات ذهبن ذهبوا الجامعة.",
+          "suggestions": [
+            {
+              "alternatives": [],
+              "confidence": 1.0,
+              "correction": "ذهبن",
+              "end": 16,
+              "id": "e9626053-e05b-4774-bd33-2155ee6d7fba",
+              "locked": true,
+              "original": "ذهبو",
+              "priority": 3,
+              "start": 12,
+              "type": "grammar"
+            },
+            {
+              "alternatives": [],
+              "confidence": 0.8,
+              "correction": "الجامعة.",
+              "end": 28,
+              "id": "4ecab998-db9d-47b5-a835-a4516a38b1ae",
+              "locked": true,
+              "original": "الجامعه",
+              "priority": 2,
+              "start": 21,
+              "type": "punctuation"
+            },
+            {
+              "alternatives": [],
+              "confidence": 1.0,
+              "correction": "إن",
+              "end": 2,
+              "id": "864a48a7-d61a-4c9e-8953-72826c279d48",
+              "locked": true,
+              "original": "ان",
+              "priority": 1,
+              "start": 0,
+              "type": "spelling"
+            },
+            {
+              "alternatives": [
+                "ذهبوا",
+                "ال",
+                "الم",
+                "الى"
+              ],
+              "confidence": 0.9,
+              "correction": "ذهبوا",
+              "end": 20,
+              "id": "05c65f7f-14f4-474c-bff8-0ce52ce5cf5b",
+              "locked": true,
+              "original": "الى",
+              "priority": 1,
+              "start": 17,
+              "type": "spelling"
+            }
+          ]
+        }
+      ],
+      "deterministic": true
+    }
+  ],
+  "boundary_tests": [
+    {
+      "id": "BOUND-299",
+      "category": 3,
+      "input_len": 299,
+      "input": "يستخدم الذكاء الاصطناعي تقنيات التعلم العميق في معالجة البيانات يستخدم الذكاء ال...",
+      "has_spelling_suggestions": false,
+      "total_suggestions": 6,
+      "timing": {
+        "grammar_ms": 5256,
+        "punctuation_ms": 5490,
+        "spelling_ms": 32835,
+        "total_ms": 43584
+      }
+    },
+    {
+      "id": "BOUND-300",
+      "category": 3,
+      "input_len": 300,
+      "input": "يستخدم الذكاء الاصطناعي تقنيات التعلم العميق في معالجة البيانات يستخدم الذكاء ال...",
+      "has_spelling_suggestions": false,
+      "total_suggestions": 9,
+      "timing": {
+        "grammar_ms": 11035,
+        "punctuation_ms": 5849,
+        "spelling_ms": 18786,
+        "total_ms": 35674
+      }
+    },
+    {
+      "id": "BOUND-301",
+      "category": 3,
+      "input_len": 301,
+      "input": "يستخدم الذكاء الاصطناعي تقنيات التعلم العميق في معالجة البيانات يستخدم الذكاء ال...",
+      "has_spelling_suggestions": false,
+      "total_suggestions": 9,
+      "timing": {
+        "grammar_ms": 12363,
+        "punctuation_ms": 6256,
+        "spelling_ms": 3209,
+        "total_ms": 21833
+      }
+    },
+    {
+      "id": "BOUND-500",
+      "category": 3,
+      "input_len": 500,
+      "input": "يستخدم الذكاء الاصطناعي تقنيات التعلم العميق في معالجة البيانات يستخدم الذكاء ال...",
+      "has_spelling_suggestions": false,
+      "total_suggestions": 23,
+      "timing": {
+        "grammar_ms": 18635,
+        "punctuation_ms": 12917,
+        "spelling_ms": 0,
+        "total_ms": 31560
+      }
+    }
+  ]
+}

tests/deep_dive_test.py

@@ -0,0 +1,519 @@
+"""
+BAYAN Deep-Dive Test Harness — Track A (Raw Models via API) & Track B (Full Pipeline via API)
+
+Uses the deployed HF Space API (bayan10/bayan-api) instead of loading models locally.
+This avoids the 1GB model download hang and tests the ACTUAL production behavior.
+
+Track A: /api/spelling, /api/grammar, /api/punctuation (individual model endpoints)
+Track B: /api/analyze (full pipeline with StageLocker, OffsetMapper, PatchSet)
+
+Usage:
+    python tests/deep_dive_test.py --stage spelling
+    python tests/deep_dive_test.py --stage grammar
+    python tests/deep_dive_test.py --stage punctuation
+    python tests/deep_dive_test.py --stage pipeline
+    python tests/deep_dive_test.py --stage all
+"""
+
+import sys, os, re, json, time, argparse
+from datetime import datetime, timezone
+
+# ═══════════════════════════════════════════════════════════════════
+# API CLIENT
+# ═══════════════════════════════════════════════════════════════════
+
+import requests
+
+API_BASE = "https://bayan10-bayan-api.hf.space"
+TIMEOUT = 60  # seconds per request
+
+def api_call(endpoint, text, retries=2):
+    """Call the deployed API with retry."""
+    url = f"{API_BASE}{endpoint}"
+    for attempt in range(retries + 1):
+        try:
+            t0 = time.time()
+            resp = requests.post(url, json={"text": text}, timeout=TIMEOUT)
+            elapsed = int((time.time() - t0) * 1000)
+            if resp.status_code == 200:
+                data = resp.json()
+                data['_elapsed_ms'] = elapsed
+                data['_timestamp'] = datetime.now(timezone.utc).isoformat()
+                return data
+            else:
+                if attempt < retries:
+                    time.sleep(2)
+                    continue
+                return {"error": f"HTTP {resp.status_code}: {resp.text[:200]}", "_elapsed_ms": elapsed}
+        except requests.exceptions.Timeout:
+            if attempt < retries:
+                time.sleep(2)
+                continue
+            return {"error": f"Timeout after {TIMEOUT}s", "_elapsed_ms": TIMEOUT * 1000}
+        except Exception as e:
+            return {"error": str(e)}
+
+# ═══════════════════════════════════════════════════════════════════
+# TRACK A — RAW MODEL CALLS (individual endpoints, no pipeline)
+# ═══════════════════════════════════════════════════════════════════
+
+def track_a_spelling(text):
+    """Call /api/spelling — raw AraSpell output."""
+    result = api_call("/api/spelling", text)
+    if "error" in result:
+        return {"input": text, "output": text, "error": result["error"], "changed": False}
+    corrected = result.get("corrected_text", text)
+    return {
+        "input": text, "output": corrected, "changed": corrected != text,
+        "elapsed_ms": result.get("_elapsed_ms"), "timestamp": result.get("_timestamp")
+    }
+
+def track_a_grammar(text):
+    """Call /api/grammar — raw grammar model output."""
+    result = api_call("/api/grammar", text)
+    if "error" in result:
+        return {"input": text, "output": text, "error": result["error"], "changed": False}
+    corrected = result.get("corrected_text", text)
+    return {
+        "input": text, "output": corrected, "changed": corrected != text,
+        "elapsed_ms": result.get("_elapsed_ms"), "timestamp": result.get("_timestamp")
+    }
+
+def track_a_punctuation(text):
+    """Call /api/punctuation — raw PuncAra output."""
+    result = api_call("/api/punctuation", text)
+    if "error" in result:
+        return {"input": text, "output": text, "error": result["error"], "changed": False}
+    corrected = result.get("corrected_text", text)
+    marks_before = sum(1 for c in text if c in '.,;:!?،؛؟')
+    marks_after = sum(1 for c in corrected if c in '.,;:!?،؛؟')
+    return {
+        "input": text, "output": corrected, "changed": corrected != text,
+        "marks_added": marks_after - marks_before,
+        "elapsed_ms": result.get("_elapsed_ms"), "timestamp": result.get("_timestamp")
+    }
+
+# ═══════════════════════════════════════════════════════════════════
+# TRACK B — FULL PIPELINE (/api/analyze)
+# ═══════════════════════════════════════════════════════════════════
+
+def track_b_analyze(text):
+    """Call /api/analyze — full pipeline with all stages."""
+    result = api_call("/api/analyze", text)
+    if "error" in result and "status" not in result:
+        return {"input": text, "error": result["error"], "suggestions": []}
+    return {
+        "input": text,
+        "original": result.get("original", text),
+        "corrected": result.get("corrected", text),
+        "suggestions": result.get("suggestions", []),
+        "timing_ms": result.get("timing_ms", {}),
+        "elapsed_ms": result.get("_elapsed_ms"),
+        "timestamp": result.get("_timestamp"),
+    }
+
+# ═══════════════════════════════════════════════════════════════════
+# TEST INPUTS — ALL CATEGORIES
+# ═══════════════════════════════════════════════════════════════════
+
+CAT2_OVERCORRECTION = [
+    {"id": "C2-01", "input": "القاهرة عاصمة جمهورية مصر العربية وأكبر مدنها", "domain": "news"},
+    {"id": "C2-02", "input": "يعد نهر النيل أطول أنهار العالم", "domain": "news"},
+    {"id": "C2-03", "input": "بسم الله الرحمن الرحيم", "domain": "religious"},
+    {"id": "C2-04", "input": "إنا لله وإنا إليه راجعون", "domain": "religious"},
+    {"id": "C2-05", "input": "يستخدم الذكاء الاصطناعي تقنيات التعلم العميق", "domain": "technical"},
+    {"id": "C2-06", "input": "سافر محمد إلى دبي للعمل في شركة جوجل", "domain": "proper_nouns"},
+    {"id": "C2-07", "input": "الرئيس عبد الفتاح السيسي رئيس جمهورية مصر العربية", "domain": "proper_nouns"},
+    {"id": "C2-08", "input": "استوقفني المشهد فتأملته مليا", "domain": "literary"},
+    {"id": "C2-09", "input": "أضحى التعليم الإلكتروني ضرورة ملحة في عصرنا الحالي", "domain": "formal"},
+    {"id": "C2-10", "input": "تتراوح درجات الحرارة بين خمس وعشرين وثلاثين درجة مئوية", "domain": "weather"},
+]
+
+CAT8_CLITIC_ROOTS = [
+    ('مدرسة', 'moon'),       # Moon letter
+    ('شمس', 'sun'),          # Sun letter
+    ('أمة', 'hamza'),        # Hamza-initial
+    ('نافذة', 'long'),       # Long word
+    ('علم', 'short'),        # Short 3-letter root
+    ('اقتصاد', 'alef'),     # Alef-initial, long
+]
+CAT8_PREFIXES = [("bare", ""), ("wa", "و"), ("ba", "ب"), ("la", "ل"), ("ka", "ك")]
+CAT8_TESTS = []
+for root, root_type in CAT8_CLITIC_ROOTS:
+    for pfx_name, pfx in CAT8_PREFIXES:
+        word = pfx + root
+        CAT8_TESTS.append({
+            "id": f"C8-{root}-{pfx_name}", "input": word, "root": root,
+            "root_type": root_type, "prefix": pfx, "expected": word,
+        })
+
+CAT9_CONFUSABLE = [
+    # === Isolation tests ===
+    {"id": "C9-01a", "input": "ان", "context": "isolation", "concern": "should→أن/إن NOT كان"},
+    {"id": "C9-01b", "input": "كان", "context": "isolation", "concern": "stays كان"},
+    {"id": "C9-02a", "input": "إلى", "context": "isolation", "concern": "stays إلى"},
+    {"id": "C9-02b", "input": "على", "context": "isolation", "concern": "stays على"},
+    {"id": "C9-03a", "input": "هذا", "context": "isolation", "concern": "stays هذا"},
+    {"id": "C9-03b", "input": "هذه", "context": "isolation", "concern": "stays هذه"},
+    {"id": "C9-03c", "input": "هذة", "context": "isolation", "concern": "misspelling→هذه"},
+    {"id": "C9-04a", "input": "لكن", "context": "isolation", "concern": "stays لكن"},
+    {"id": "C9-04b", "input": "لاكن", "context": "isolation", "concern": "misspelling→لكن"},
+    {"id": "C9-05a", "input": "ذلك", "context": "isolation", "concern": "stays ذلك"},
+    {"id": "C9-05b", "input": "ذالك", "context": "isolation", "concern": "misspelling→ذلك"},
+    {"id": "C9-06a", "input": "الى", "context": "isolation", "concern": "should→إلى"},
+    # === Sentence-context tests ===
+    {"id": "C9-S01", "input": "ان الحياة جميلة", "context": "sentence", "concern": "ان→أن/إن NOT كان"},
+    {"id": "C9-S02", "input": "كان الرجل طيبا", "context": "sentence", "concern": "كان stays"},
+    {"id": "C9-S03", "input": "ذهب الى المدرسة", "context": "sentence", "concern": "الى→إلى"},
+    {"id": "C9-S04", "input": "جلس على الكرسي", "context": "sentence", "concern": "على stays"},
+    {"id": "C9-S05", "input": "هذة المدينة جميلة", "context": "sentence", "concern": "هذة→هذه"},
+    {"id": "C9-S06", "input": "هو ذكي لاكن كسول", "context": "sentence", "concern": "لاكن→لكن"},
+    {"id": "C9-S07", "input": "ذالك الكتاب مفيد", "context": "sentence", "concern": "ذالك→ذلك"},
+    {"id": "C9-S08", "input": "هذا البيت كبير", "context": "sentence", "concern": "هذا stays"},
+    {"id": "C9-S09", "input": "هذه السيارة سريعة", "context": "sentence", "concern": "هذه stays"},
+    {"id": "C9-S10", "input": "سافر إلى القاهرة", "context": "sentence", "concern": "إلى stays"},
+    {"id": "C9-S11", "input": "جلس على المقعد", "context": "sentence", "concern": "على stays"},
+    {"id": "C9-S12", "input": "ان الذكاء مهم لكن الاجتهاد اهم", "context": "sentence", "concern": "ان→أن, لكن stays"},
+]
+
+CAT10_EDGE_CASES = [
+    {"id": "C10-01", "input": "كَتَبَ الطَّالِبُ الدَّرسَ", "concern": "tashkeel_present"},
+    {"id": "C10-02", "input": "كتب الطالب الدرس", "concern": "tashkeel_absent"},
+    {"id": "C10-03", "input": "قرأ إبراهيم آيات من القرآن", "concern": "alef_forms"},
+    {"id": "C10-04", "input": "مشى الفتى إلى المستشفى", "concern": "ya_alef_maksura"},
+    {"id": "C10-05", "input": "ذهبت إلى المدرسة", "concern": "ta_marbuta"},
+    {"id": "C10-06", "input": "جاء ١٢٣ طالبا", "concern": "arabic_indic_digits"},
+    {"id": "C10-07", "input": "جاء 123 طالبا", "concern": "western_digits"},
+    {"id": "C10-08", "input": "يعمل في شركة Google في القاهرة", "concern": "latin_in_arabic"},
+    {"id": "C10-09", "input": "انا رايح المدرسة النهارده", "concern": "egyptian_dialect"},
+    {"id": "C10-10", "input": "الموضوع ده كويس جدااااا", "concern": "repeated_letters"},
+    {"id": "C10-11", "input": "مسؤول عن الشؤون الداخلية", "concern": "hamza_on_waw"},
+    {"id": "C10-12", "input": "بيئة العمل مليئة بالتحديات", "concern": "hamza_on_ya"},
+    {"id": "C10-13", "input": "الكتاب الذى قرأته مفيد", "concern": "ya_in_الذي"},
+    {"id": "C10-14", "input": "خطأ الطالب في الامتحان", "concern": "hamza_standalone"},
+    {"id": "C10-15", "input": "الحمد لله رب العالمين الرحمن الرحيم مالك يوم الدين", "concern": "religious_long"},
+]
+
+CAT5_PUNC_SANITY = [
+    {"id": "C5-01", "input": "ذهب إلى المدرسة", "length": "short_3w"},
+    {"id": "C5-02", "input": "هل تعلم أن الأرض تدور حول الشمس كل عام", "length": "medium_9w"},
+    {"id": "C5-03", "input": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة في بناء كتلة عضلية قوية ويا له من التزام حديدي يثير الإعجاب", "length": "long_20w"},
+    {"id": "C5-04", "input": "قال المعلم للطلاب ادرسوا جيدا فالامتحان قريب", "length": "medium_imperative"},
+    {"id": "C5-05", "input": "كانت الفتيات يلعبن في الحديقة وفجأة سقطت إحداهن وبدأت تبكي بشدة", "length": "long_narrative"},
+]
+
+CAT6_PUNC_POSITION = [
+    {"id": "C6-01", "input": "ذهب محمد إلى المدرسة ودرس جيدا ثم عاد إلى البيت"},
+    {"id": "C6-02", "input": "إن الذكاء الاصطناعي يلعب دورا هاما لذلك يجب الاهتمام به"},
+    {"id": "C6-03", "input": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة في بناء كتلة عضلية قوية ويا له من التزام حديدي يثير الإعجاب"},
+    {"id": "C6-04", "input": "كانت الفتيات يلعبن في الحديقة وفجأة سقطت إحداهن وبدأت تبكي بشدة"},
+    {"id": "C6-05", "input": "هل تعلم أن القاهرة هي عاصمة مصر وتقع على ضفاف نهر النيل"},
+    {"id": "C6-06", "input": "قال المعلم للطلاب ادرسوا جيدا فالامتحان قريب"},
+    {"id": "C6-07", "input": "يحب الأطفال اللعب في الحديقة وركوب الدراجات والجري بين الأشجار"},
+    {"id": "C6-08", "input": "رغم صعوبة الامتحان إلا أن الطلاب حققوا نتائج مبهرة"},
+    {"id": "C6-09", "input": "سافر العالم إلى عدة دول لحضور المؤتمرات العلمية ونشر أبحاثه"},
+    {"id": "C6-10", "input": "يا بني اجتهد في دراستك فالعلم نور والجهل ظلام"},
+]
+
+# ═══════════════════════════════════════════════════════════════════
+# RUNNERS
+# ═══════════════════════════════════════════════════════════════════
+
+def log(msg):
+    print(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}", flush=True)
+
+def run_spelling_tests():
+    results = []
+
+    log("=== Category 2: Overcorrection (10 tests) ===")
+    for test in CAT2_OVERCORRECTION:
+        log(f"  {test['id']}: {test['input'][:50]}...")
+        a = track_a_spelling(test['input'])
+        b = track_b_analyze(test['input'])
+        fp = a.get('changed', False)
+        result = {
+            "id": test['id'], "category": 2, "input": test['input'],
+            "domain": test['domain'],
+            "track_a_spelling": a['output'],
+            "track_a_changed": a.get('changed', False),
+            "track_b_suggestions": len(b.get('suggestions', [])),
+            "track_b_corrected": b.get('corrected', ''),
+            "is_false_positive": fp,
+        }
+        status = "⚠ FP" if fp else "✓"
+        log(f"    {status} A:'{a['output'][:60]}' B_sugg:{len(b.get('suggestions',[]))}")
+        results.append(result)
+
+    log("\n=== Category 8: Clitic/Prefix (30 tests) ===")
+    for test in CAT8_TESTS:
+        a = track_a_spelling(test['input'])
+        changed = a.get('changed', False)
+        if changed:
+            # Classify: did it preserve root or mangle it?
+            output = a['output']
+            root_preserved = test['root'] in output or any(
+                test['root'][:-1] in output  # partial root match
+                for _ in [1]
+            )
+            classification = "root_fixed" if root_preserved else "prefix_mangled"
+        else:
+            classification = "correct"
+        result = {
+            "id": test['id'], "category": 8, "input": test['input'],
+            "root": test['root'], "root_type": test['root_type'],
+            "prefix": test['prefix'],
+            "track_a_spelling": a['output'], "changed": changed,
+            "classification": classification,
+        }
+        if changed:
+            log(f"  ⚠ {test['id']}: '{test['input']}' → '{a['output']}' [{classification}]")
+        results.append(result)
+
+    log("\n=== Category 9: Confusable Words (24 tests) ===")
+    for test in CAT9_CONFUSABLE:
+        a = track_a_spelling(test['input'])
+        result = {
+            "id": test['id'], "category": 9, "input": test['input'],
+            "context": test['context'], "concern": test['concern'],
+            "track_a_spelling": a['output'], "changed": a.get('changed', False),
+        }
+        if a.get('changed'):
+            log(f"  ⚠ {test['id']}: '{test['input']}' → '{a['output']}' (concern: {test['concern']})")
+        else:
+            log(f"  ✓ {test['id']}: no change")
+        results.append(result)
+
+    log("\n=== Category 10: Arabic Edge Cases (15 tests) ===")
+    for test in CAT10_EDGE_CASES:
+        a = track_a_spelling(test['input'])
+        result = {
+            "id": test['id'], "category": 10, "input": test['input'],
+            "concern": test['concern'],
+            "track_a_spelling": a['output'], "changed": a.get('changed', False),
+        }
+        if a.get('changed'):
+            log(f"  ⚠ {test['id']}: '{test['input']}' → '{a['output']}' [{test['concern']}]")
+        else:
+            log(f"  ✓ {test['id']}: no change [{test['concern']}]")
+        results.append(result)
+
+    return results
+
+def run_punctuation_tests():
+    results = []
+
+    log("=== Category 5: Punctuation Sanity (5 tests) ===")
+    for test in CAT5_PUNC_SANITY:
+        log(f"  {test['id']}: {test['input'][:50]}...")
+        a = track_a_punctuation(test['input'])
+        result = {
+            "id": test['id'], "category": 5, "input": test['input'],
+            "length": test['length'],
+            "track_a_punc": a['output'],
+            "marks_added": a.get('marks_added', 0),
+            "changed": a.get('changed', False),
+        }
+        log(f"    Marks: +{a.get('marks_added', 0)} | Output: {a['output'][:80]}")
+        results.append(result)
+
+    log("\n=== Category 6: Punctuation Position (10 tests) ===")
+    for test in CAT6_PUNC_POSITION:
+        log(f"  {test['id']}: {test['input'][:50]}...")
+        # Track A: raw punctuation on original text
+        a_punc = track_a_punctuation(test['input'])
+        # Track B: full pipeline
+        b = track_b_analyze(test['input'])
+
+        # Measure: where did Track A put punctuation marks?
+        a_marks = _find_punct_positions(test['input'], a_punc['output'])
+        # Measure: where did Track B put punctuation suggestions?
+        b_punc_sugg = [s for s in b.get('suggestions', []) if s.get('type') == 'punctuation']
+        b_marks = [(s.get('start', 0), s.get('end', 0), s.get('correction', '')) for s in b_punc_sugg]
+
+        result = {
+            "id": test['id'], "category": 6, "input": test['input'],
+            "track_a_punc_output": a_punc['output'],
+            "track_a_marks": a_marks,
+            "track_b_corrected": b.get('corrected', ''),
+            "track_b_punc_suggestions": b_punc_sugg,
+            "track_b_marks": b_marks,
+        }
+        log(f"    A marks: {a_marks}")
+        log(f"    B marks: {b_marks}")
+        results.append(result)
+
+    return results
+
+def _find_punct_positions(original, punctuated):
+    """Find where punctuation was added by comparing original vs punctuated."""
+    PUNC = set('.,;:!?،؛؟')
+    marks = []
+    # Word-level alignment
+    orig_words = original.split()
+    punc_words = punctuated.split()
+    oi, pi = 0, 0
+    char_pos = 0
+    while oi < len(orig_words) and pi < len(punc_words):
+        o_base = ''.join(c for c in orig_words[oi] if c not in PUNC)
+        p_base = ''.join(c for c in punc_words[pi] if c not in PUNC)
+        if o_base == p_base:
+            # Same word — check for added punctuation
+            o_punc = set(c for c in orig_words[oi] if c in PUNC)
+            p_punc = set(c for c in punc_words[pi] if c in PUNC)
+            added = p_punc - o_punc
+            if added:
+                marks.append({
+                    "word_index": oi, "word": orig_words[oi],
+                    "after_word": orig_words[oi],
+                    "marks_added": list(added),
+                    "char_pos": char_pos,
+                })
+            char_pos += len(orig_words[oi]) + 1  # +1 for space
+            oi += 1
+            pi += 1
+        else:
+            # Mismatch — model changed the word
+            char_pos += len(orig_words[oi]) + 1
+            oi += 1
+            pi += 1
+    return marks
+
+def run_pipeline_comparison():
+    """Run tests that need both Track A and Track B for comparison (Cat 1, 3, 4, 7)."""
+    results = []
+
+    # Cat 3: Integration-only — test where raw models work but pipeline might not
+    log("=== Category 3: Integration-Only (5 tests) ===")
+    integration_inputs = [
+        {"id": "C3-01", "input": "كانت الفتيات يلعبون في الحديقه وفجأه سقطت احداهن وبدءت تبكي بشده"},
+        {"id": "C3-02", "input": "ان الذكاء الاصطناعي يلعب دورا هاما ولذالك يجب الاهتمام بة"},
+        {"id": "C3-03", "input": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة في بناء كتلة عضلية قوية ويا له من التزام حديدي يثير الإعجاب"},
+        {"id": "C3-04", "input": "هذة المدينه جميله جدا ومناخها معتدل طوال العام"},
+        {"id": "C3-05", "input": "الطلاب الذين اجتهدو في دراستهم حققو نتائج ممتازه في الامتحانات"},
+    ]
+    for test in integration_inputs:
+        log(f"  {test['id']}: {test['input'][:50]}...")
+        a_spell = track_a_spelling(test['input'])
+        a_gram = track_a_grammar(test['input'])
+        a_punc = track_a_punctuation(test['input'])
+        b = track_b_analyze(test['input'])
+        result = {
+            "id": test['id'], "category": 3, "input": test['input'],
+            "track_a": {
+                "spelling": a_spell['output'], "spelling_changed": a_spell.get('changed'),
+                "grammar": a_gram['output'], "grammar_changed": a_gram.get('changed'),
+                "punctuation": a_punc['output'], "punctuation_changed": a_punc.get('changed'),
+            },
+            "track_b": {
+                "corrected": b.get('corrected', ''),
+                "suggestions": b.get('suggestions', []),
+                "timing_ms": b.get('timing_ms', {}),
+            }
+        }
+        log(f"    A_spell: {a_spell['output'][:60]}")
+        log(f"    A_gram:  {a_gram['output'][:60]}")
+        log(f"    A_punc:  {a_punc['output'][:60]}")
+        log(f"    B_final: {b.get('corrected','')[:60]}")
+        log(f"    B_sugg:  {len(b.get('suggestions',[]))}")
+        results.append(result)
+
+    # Cat 4: Overlap — run 3x for determinism
+    log("\n=== Category 4: Overlap Resolution (3 tests × 3 runs) ===")
+    overlap_inputs = [
+        {"id": "C4-01", "input": "كانت الفتيات يلعبون في الحديقه"},
+        {"id": "C4-02", "input": "ذهب الى المدرسه وقابل المعلمه"},
+        {"id": "C4-03", "input": "ان الطالبات ذهبو الى الجامعه"},
+    ]
+    for test in overlap_inputs:
+        runs = []
+        for run_idx in range(3):
+            b = track_b_analyze(test['input'])
+            runs.append({
+                "run": run_idx + 1,
+                "corrected": b.get('corrected', ''),
+                "suggestions": b.get('suggestions', []),
+            })
+        # Check determinism
+        all_same = all(r['corrected'] == runs[0]['corrected'] for r in runs)
+        result = {
+            "id": test['id'], "category": 4, "input": test['input'],
+            "runs": runs, "deterministic": all_same,
+        }
+        log(f"  {test['id']}: deterministic={all_same}")
+        for r in runs:
+            log(f"    Run {r['run']}: {r['corrected'][:60]} ({len(r['suggestions'])} sugg)")
+        results.append(result)
+
+    return results
+
+# Boundary tests for spelling 300-char cutoff
+def run_boundary_tests():
+    results = []
+    log("\n=== Boundary: Spelling 300-char cutoff ===")
+    base = "يستخدم الذكاء الاصطناعي تقنيات التعلم العميق في معالجة البيانات "
+    for target_len in [299, 300, 301, 500]:
+        text = (base * 10)[:target_len]
+        b = track_b_analyze(text)
+        has_spelling = any(s.get('type') == 'spelling' for s in b.get('suggestions', []))
+        result = {
+            "id": f"BOUND-{target_len}", "category": 3, "input_len": target_len,
+            "input": text[:80] + "...",
+            "has_spelling_suggestions": has_spelling,
+            "total_suggestions": len(b.get('suggestions', [])),
+            "timing": b.get('timing_ms', {}),
+        }
+        log(f"  len={target_len}: spelling_active={has_spelling} suggestions={len(b.get('suggestions',[]))}")
+        results.append(result)
+    return results
+
+# ═══════════════════════════════════════════════════════════════════
+# MAIN
+# ═══════════════════════════════════════════════════════════════════
+
+def main():
+    parser = argparse.ArgumentParser(description='BAYAN Deep-Dive Test Harness')
+    parser.add_argument('--stage', choices=['spelling', 'grammar', 'punctuation', 'pipeline', 'all'],
+                       default='spelling')
+    args = parser.parse_args()
+
+    all_results = {"timestamp": datetime.now(timezone.utc).isoformat(), "api_base": API_BASE}
+
+    # Health check
+    log(f"Checking API health at {API_BASE}...")
+    try:
+        resp = requests.get(f"{API_BASE}/api/health", timeout=10)
+        log(f"  Health: {resp.status_code} — {resp.json()}")
+        all_results['health'] = resp.json()
+    except Exception as e:
+        log(f"  ⚠ API unreachable: {e}")
+        all_results['health'] = {"error": str(e)}
+
+    if args.stage in ('spelling', 'all'):
+        log("\n══════ SPELLING TESTS (Cat 2, 8, 9, 10) ══════")
+        all_results['spelling_tests'] = run_spelling_tests()
+
+    if args.stage in ('punctuation', 'all'):
+        log("\n══════ PUNCTUATION TESTS (Cat 5, 6) ══════")
+        all_results['punctuation_tests'] = run_punctuation_tests()
+
+    if args.stage in ('pipeline', 'all'):
+        log("\n══════ PIPELINE TESTS (Cat 3, 4) ══════")
+        all_results['pipeline_tests'] = run_pipeline_comparison()
+        all_results['boundary_tests'] = run_boundary_tests()
+
+    # Save
+    output_path = os.path.join(os.path.dirname(__file__), 'deep_dive_output.json')
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(all_results, f, ensure_ascii=False, indent=2)
+    log(f"\nResults saved to {output_path}")
+
+    # Summary
+    for key in ['spelling_tests', 'punctuation_tests', 'pipeline_tests', 'boundary_tests']:
+        if key in all_results:
+            tests = all_results[key]
+            if isinstance(tests, list):
+                changed = sum(1 for t in tests if t.get('changed') or t.get('is_false_positive'))
+                log(f"  {key}: {len(tests)} tests, {changed} with changes")
+
+if __name__ == '__main__':
+    main()

tests/gap_filling_results.json

@@ -0,0 +1,261 @@
+{
+  "phase_1_3": [
+    {
+      "input": "لكن الأمر مختلف",
+      "corrected": "لكن الأمر مختلف.",
+      "check": "لكن",
+      "status": "✅ PRESERVED",
+      "suggestions": 1
+    },
+    {
+      "input": "ذلك الكتاب جميل",
+      "corrected": "ذلك الكتاب جميل.",
+      "check": "ذلك",
+      "status": "✅ PRESERVED",
+      "suggestions": 1
+    },
+    {
+      "input": "إلى المدرسة",
+      "corrected": "إلى المدرسة.",
+      "check": "إلى",
+      "status": "✅ PRESERVED",
+      "suggestions": 1
+    },
+    {
+      "input": "على الطاولة",
+      "corrected": "على الطاولة.",
+      "check": "على",
+      "status": "✅ PRESERVED",
+      "suggestions": 1
+    },
+    {
+      "input": "هذه المدينة جميلة",
+      "corrected": "هذه المدينة جميلة.",
+      "check": "هذه",
+      "status": "✅ PRESERVED",
+      "suggestions": 1
+    },
+    {
+      "input": "كان الجو حارا",
+      "corrected": "كان الجو حارا.",
+      "check": "كان",
+      "status": "✅ PRESERVED",
+      "suggestions": 1
+    },
+    {
+      "input": "لاكن الأمر مختلف",
+      "corrected": "لكن الأمر مختلف.",
+      "check": "لاكن→لكن",
+      "status": "✅ CORRECTED",
+      "suggestions": 2
+    },
+    {
+      "input": "ذالك الكتاب جميل",
+      "corrected": "ذلك الكتاب جميل.",
+      "check": "ذالك→ذلك",
+      "status": "✅ CORRECTED",
+      "suggestions": 2
+    }
+  ],
+  "phase_2": {
+    "total": 10,
+    "raw_fp_count": 5,
+    "raw_fp_rate": "50%",
+    "pipeline_fp_count": 1,
+    "pipeline_fp_rate": "10%",
+    "results": [
+      {
+        "id": "R-01",
+        "word": "عصماء",
+        "raw_changed": true,
+        "pipeline_changed": false,
+        "pipeline_targeted": false,
+        "is_false_positive": false
+      },
+      {
+        "id": "R-02",
+        "word": "يستشف",
+        "raw_changed": true,
+        "pipeline_changed": false,
+        "pipeline_targeted": false,
+        "is_false_positive": false
+      },
+      {
+        "id": "R-03",
+        "word": "المسغبة",
+        "raw_changed": true,
+        "pipeline_changed": false,
+        "pipeline_targeted": false,
+        "is_false_positive": false
+      },
+      {
+        "id": "R-04",
+        "word": "التقاعس",
+        "raw_changed": true,
+        "pipeline_changed": false,
+        "pipeline_targeted": false,
+        "is_false_positive": false
+      },
+      {
+        "id": "R-05",
+        "word": "استئثار",
+        "raw_changed": false,
+        "pipeline_changed": false,
+        "pipeline_targeted": false,
+        "is_false_positive": false
+      },
+      {
+        "id": "R-06",
+        "word": "تبجيل",
+        "raw_changed": false,
+        "pipeline_changed": false,
+        "pipeline_targeted": false,
+        "is_false_positive": false
+      },
+      {
+        "id": "R-07",
+        "word": "الدمث",
+        "raw_changed": true,
+        "pipeline_changed": true,
+        "pipeline_targeted": true,
+        "is_false_positive": true
+      },
+      {
+        "id": "R-08",
+        "word": "استقصاء",
+        "raw_changed": false,
+        "pipeline_changed": false,
+        "pipeline_targeted": false,
+        "is_false_positive": false
+      },
+      {
+        "id": "R-09",
+        "word": "التواني",
+        "raw_changed": false,
+        "pipeline_changed": false,
+        "pipeline_targeted": false,
+        "is_false_positive": false
+      },
+      {
+        "id": "R-10",
+        "word": "مستطرف",
+        "raw_changed": false,
+        "pipeline_changed": false,
+        "pipeline_targeted": false,
+        "is_false_positive": false
+      }
+    ]
+  },
+  "phase_3_2": [
+    {
+      "input": "ولذالك قررت السفر",
+      "corrected": "ولذالك قررت السفر.",
+      "bad_split_present": false,
+      "good_correction_present": false
+    },
+    {
+      "input": "المستشفياتهم كبيرة",
+      "corrected": "المستشفيات هم كبيرة.",
+      "bad_split_present": false,
+      "good_correction_present": false
+    }
+  ],
+  "phase_5_5": [
+    {
+      "input": "الطالبه كتبو الوجبات",
+      "corrected": "الطالبة كتبو الوجبات.",
+      "suggestions": 2,
+      "has_duplicate": false,
+      "word_count_diff": 0
+    },
+    {
+      "input": "هو ذهبو الي البيت",
+      "corrected": "هو ذهب إلى البيت.",
+      "suggestions": 3,
+      "has_duplicate": false,
+      "word_count_diff": 0
+    },
+    {
+      "input": "الطلاب اجتهدو في امتحانتهم",
+      "corrected": "الطلاب اجتهدو في امتحانتهم.",
+      "suggestions": 1,
+      "has_duplicate": false,
+      "word_count_diff": 0
+    }
+  ],
+  "phase_6_3": {
+    "empty_count": 0,
+    "error_count": 0,
+    "results": [
+      {
+        "attempt": 1,
+        "corrected": "الحديقة جميلة والأزهار متفتحة.",
+        "suggestions": 2,
+        "status": "success",
+        "warnings": {},
+        "is_empty": false,
+        "is_error": false
+      },
+      {
+        "attempt": 2,
+        "corrected": "الحديقة جميلة والأزهار متفتحة.",
+        "suggestions": 2,
+        "status": "success",
+        "warnings": {},
+        "is_empty": false,
+        "is_error": false
+      },
+      {
+        "attempt": 3,
+        "corrected": "الحديقة جميلة والأزهار متفتحة.",
+        "suggestions": 2,
+        "status": "success",
+        "warnings": {},
+        "is_empty": false,
+        "is_error": false
+      },
+      {
+        "attempt": 4,
+        "corrected": "الحديقة جميلة والأزهار متفتحة.",
+        "suggestions": 2,
+        "status": "success",
+        "warnings": {},
+        "is_empty": false,
+        "is_error": false
+      },
+      {
+        "attempt": 5,
+        "corrected": "الحديقة جميلة والأزهار متفتحة.",
+        "suggestions": 2,
+        "status": "success",
+        "warnings": {},
+        "is_empty": false,
+        "is_error": false
+      }
+    ]
+  },
+  "phase_6_4": {
+    "input_chars": 982,
+    "input_words": 159,
+    "status": "success",
+    "suggestions": 4,
+    "warnings": {},
+    "timing": {
+      "grammar_ms": 12196,
+      "punctuation_ms": 14448,
+      "spelling_ms": 0,
+      "total_ms": 26649
+    },
+    "elapsed_ms": 27615,
+    "is_silently_empty": false
+  },
+  "phase_7_1": {
+    "input": "قال المعلم للطلاب ادرسوا جيدا فالامتحان قريب",
+    "raw_output": "قال المعلم للطلاب ادرسوا: جيدا فالامتحان قريب؛",
+    "pipeline_output": "قال المعلم للطلاب ادرسوا: جيدا فالامتحين قريب",
+    "has_semicolon_raw": true,
+    "has_semicolon_pipeline": false,
+    "diagnosis": "StageLocker or validate_punctuation_diff rejection",
+    "pipeline_punc_count": 1
+  }
+}

tests/gap_filling_tests.py

@@ -0,0 +1,522 @@
+"""
+Gap-filling live tests for all missing items from the Fix-Everything prompt.
+Covers:
+  Phase 1.3 — Category 9 pairs: لكن/لاكن, ذلك/ذالك, الى/إلى live verification
+  Phase 2   — R-01→R-10 rare vocabulary FP measurement  
+  Phase 3.2 — ولذالك and مستشفياتهم specific cases
+  Phase 5.5 — Constructed dual-correction cases
+  Phase 6.3 — BUG-017 re-test
+  Phase 6.4 — 187-word input regression
+  Phase 7.1 — BUG-018 precise tracing
+"""
+import sys, os, json, time, requests
+
+API_BASE = "https://bayan10-bayan-api.hf.space"
+TIMEOUT = 90
+
+def api_call(endpoint, text, timeout=TIMEOUT):
+    url = f"{API_BASE}{endpoint}"
+    try:
+        t0 = time.time()
+        resp = requests.post(url, json={"text": text}, timeout=timeout)
+        elapsed = int((time.time() - t0) * 1000)
+        if resp.status_code == 200:
+            data = resp.json()
+            data['_elapsed_ms'] = elapsed
+            return data
+        return {"error": f"HTTP {resp.status_code}", "_elapsed_ms": elapsed}
+    except Exception as e:
+        return {"error": f"{type(e).__name__}: {e}"}
+
+
+all_results = {}
+
+
+# ══════════════════════════════════════════════════════════════════════
+# Phase 1.3 — Category 9 Pairs Live Verification
+# ══════════════════════════════════════════════════════════════════════
+def test_category9_live():
+    print("=" * 70)
+    print("PHASE 1.3 — Category 9 Pairs Live Verification")
+    print("=" * 70)
+
+    pairs = [
+        # (input_text, word_that_must_NOT_change, description)
+        ("لكن الأمر مختلف", "لكن", "لكن must NOT become لاكن"),
+        ("ذلك الكتاب جميل", "ذلك", "ذلك must NOT become ذالك"),
+        ("إلى المدرسة", "إلى", "إلى must NOT become على"),
+        ("على الطاولة", "على", "على must NOT become إلى"),
+        ("هذه المدينة جميلة", "هذه", "هذه must NOT become هذة"),
+        ("كان الجو حارا", "كان", "كان must NOT become كأن"),
+        # Reverse direction: misspellings SHOULD be corrected
+        ("لاكن الأمر مختلف", "لاكن→لكن", "لاكن should become لكن"),
+        ("ذالك الكتاب جميل", "ذالك→ذلك", "ذالك should become ذلك"),
+    ]
+
+    results = []
+    for text, check, desc in pairs:
+        r = api_call("/api/analyze", text)
+        corrected = r.get("corrected", text)
+        suggestions = r.get("suggestions", [])
+
+        is_reverse = "→" in check
+        if is_reverse:
+            # For misspellings, check that correction happened
+            orig, expected = check.split("→")
+            if expected in corrected and orig not in corrected:
+                status = "✅ CORRECTED"
+            elif orig in corrected:
+                status = "⚠ NOT corrected (pipeline didn't fix misspelling)"
+            else:
+                status = "⚠ UNCLEAR"
+        else:
+            # For correct words, check they weren't corrupted
+            if check in corrected:
+                status = "✅ PRESERVED"
+            else:
+                status = "❌ CORRUPTED"
+
+        result = {
+            "input": text, "corrected": corrected,
+            "check": check, "status": status,
+            "suggestions": len(suggestions),
+        }
+        results.append(result)
+        print(f"\n  {desc}")
+        print(f"    Input:     '{text}'")
+        print(f"    Corrected: '{corrected}'")
+        print(f"    {status}")
+
+    return results
+
+
+# ══════════════════════════════════════════════════════════════════════
+# Phase 2 — R-01→R-10 Rare Vocabulary FP Measurement
+# ══════════════════════════════════════════════════════════════════════
+def test_rare_vocabulary():
+    print("\n" + "=" * 70)
+    print("PHASE 2 — R-01→R-10 Rare Vocabulary FP Measurement")
+    print("=" * 70)
+
+    # R-01 through R-10: valid but uncommon Arabic words
+    rare_words = [
+        {"id": "R-01", "word": "عصماء", "sentence": "المدينة العصماء تحتضن آلاف السكان",
+         "desc": "عصماء = impeccable (feminine)"},
+        {"id": "R-02", "word": "يستشف", "sentence": "يستشف الباحث نتائج الدراسة بعناية",
+         "desc": "يستشف = to discern/perceive"},
+        {"id": "R-03", "word": "المسغبة", "sentence": "أرهقته المسغبة والعطش الشديد",
+         "desc": "المسغبة = severe hunger"},
+        {"id": "R-04", "word": "التقاعس", "sentence": "التقاعس عن العمل يؤدي إلى الفشل",
+         "desc": "التقاعس = negligence/laziness"},
+        {"id": "R-05", "word": "استئثار", "sentence": "استئثار السلطة يهدد الديمقراطية",
+         "desc": "استئثار = monopolization"},
+        {"id": "R-06", "word": "تبجيل", "sentence": "تبجيل العلماء واجب على المجتمع",
+         "desc": "تبجيل = veneration"},
+        {"id": "R-07", "word": "الدمث", "sentence": "الرجل الدمث يحبه الجميع",
+         "desc": "الدمث = gentle/affable person"},
+        {"id": "R-08", "word": "استقصاء", "sentence": "استقصاء الحقائق مهم في الصحافة",
+         "desc": "استقصاء = investigation/inquiry"},
+        {"id": "R-09", "word": "التواني", "sentence": "لا يجوز التواني في طلب العلم",
+         "desc": "التواني = procrastination"},
+        {"id": "R-10", "word": "مستطرف", "sentence": "كتاب المستطرف من أمهات الكتب العربية",
+         "desc": "مستطرف = novel/curious (literary term)"},
+    ]
+
+    false_positives = 0
+    total = len(rare_words)
+    results = []
+
+    for item in rare_words:
+        # Track A: Raw spelling
+        a = api_call("/api/spelling", item["sentence"])
+        a_out = a.get("corrected_text", item["sentence"])
+        a_changed_word = item["word"] not in a_out
+
+        # Track B: Pipeline
+        b = api_call("/api/analyze", item["sentence"])
+        b_out = b.get("corrected", item["sentence"])
+        b_suggestions = b.get("suggestions", [])
+        b_changed_word = item["word"] not in b_out
+
+        # Check if any suggestion targets the rare word
+        word_targeted = False
+        targeting_suggestion = None
+        for s in b_suggestions:
+            if s.get("original", "") == item["word"]:
+                word_targeted = True
+                targeting_suggestion = s
+                break
+
+        is_fp = b_changed_word or word_targeted
+        if is_fp:
+            false_positives += 1
+
+        result = {
+            "id": item["id"],
+            "word": item["word"],
+            "raw_changed": a_changed_word,
+            "pipeline_changed": b_changed_word,
+            "pipeline_targeted": word_targeted,
+            "is_false_positive": is_fp,
+        }
+        results.append(result)
+
+        status = "❌ FALSE POSITIVE" if is_fp else "✅ PRESERVED"
+        print(f"\n  {item['id']}: {item['desc']}")
+        print(f"    Input:      '{item['sentence'][:60]}...'")
+        print(f"    Raw spell:  changed={a_changed_word}")
+        if a_changed_word:
+            print(f"    Raw output: '{a_out[:60]}...'")
+        print(f"    Pipeline:   changed={b_changed_word}, targeted={word_targeted}")
+        if b_changed_word:
+            print(f"    Pipeline:   '{b_out[:60]}...'")
+        if targeting_suggestion:
+            print(f"    Suggestion: '{targeting_suggestion.get('original','')}' → '{targeting_suggestion.get('correction','')}' (conf={targeting_suggestion.get('confidence', '?')})")
+        print(f"    {status}")
+
+    raw_fp_count = sum(1 for r in results if r["raw_changed"])
+    pipeline_fp_count = false_positives
+    print(f"\n{'=' * 50}")
+    print(f"  Raw model FP rate:  {raw_fp_count}/{total} = {raw_fp_count/total*100:.0f}%")
+    print(f"  Pipeline FP rate:   {pipeline_fp_count}/{total} = {pipeline_fp_count/total*100:.0f}%")
+
+    return {
+        "total": total,
+        "raw_fp_count": raw_fp_count,
+        "raw_fp_rate": f"{raw_fp_count/total*100:.0f}%",
+        "pipeline_fp_count": pipeline_fp_count,
+        "pipeline_fp_rate": f"{pipeline_fp_count/total*100:.0f}%",
+        "results": results,
+    }
+
+
+# ══════════════════════════════════════════════════════════════════════
+# Phase 3.2 — Specific Word-split Cases
+# ══════════════════════════════════════════════════════════════════════
+def test_word_splits():
+    print("\n" + "=" * 70)
+    print("PHASE 3.2 — Specific Word-split Verification")
+    print("=" * 70)
+
+    cases = [
+        {
+            "input": "ولذالك قررت السفر",
+            "target_word": "ولذالك",
+            "expected_correct": "ولذلك",
+            "bad_split": "ولذا ذلك",
+            "desc": "ولذالك should become ولذلك, NOT 'ولذا ذلك'"
+        },
+        {
+            "input": "المستشفياتهم كبيرة",
+            "target_word": "المستشفياتهم",
+            "expected_correct": "مستشفياتهم",
+            "bad_split": "في مستشفيات هم",
+            "desc": "مستشفياتهم should NOT be split into 'في مستشفيات هم'"
+        },
+    ]
+
+    results = []
+    for case in cases:
+        r = api_call("/api/analyze", case["input"])
+        corrected = r.get("corrected", case["input"])
+        suggestions = r.get("suggestions", [])
+
+        has_bad_split = case["bad_split"] in corrected
+        has_good_correction = case["expected_correct"] in corrected
+
+        result = {
+            "input": case["input"],
+            "corrected": corrected,
+            "bad_split_present": has_bad_split,
+            "good_correction_present": has_good_correction,
+        }
+        results.append(result)
+
+        print(f"\n  {case['desc']}")
+        print(f"    Input:     '{case['input']}'")
+        print(f"    Corrected: '{corrected}'")
+        if has_bad_split:
+            print(f"    ❌ BAD SPLIT detected: '{case['bad_split']}'")
+        elif has_good_correction:
+            print(f"    ✅ Correctly fixed to '{case['expected_correct']}'")
+        else:
+            print(f"    ⚠ Neither expected correction nor bad split found")
+
+    return results
+
+
+# ══════════════════════════════════════════════════════════════════════
+# Phase 5.5 — Constructed Dual-correction Cases
+# ══════════════════════════════════════════════════════════════════════
+def test_dual_corrections():
+    print("\n" + "=" * 70)
+    print("PHASE 5.5 — Constructed Dual-correction Cases")
+    print("=" * 70)
+
+    # Cases where spelling AND grammar would both want to change words
+    cases = [
+        {
+            "input": "الطالبه كتبو الوجبات",
+            "desc": "Spelling: الطالبه→الطالبة, Grammar: كتبو→كتبوا + possibly الوجبات→الواجبات",
+        },
+        {
+            "input": "هو ذهبو الي البيت",
+            "desc": "Spelling: الي→إلى, Grammar: ذهبو→ذهب (singular subject هو)",
+        },
+        {
+            "input": "الطلاب اجتهدو في امتحانتهم",
+            "desc": "Spelling: امتحانتهم→امتحاناتهم, Grammar: اجتهدو→اجتهدوا",
+        },
+    ]
+
+    results = []
+    for case in cases:
+        r = api_call("/api/analyze", case["input"])
+        corrected = r.get("corrected", case["input"])
+        suggestions = r.get("suggestions", [])
+
+        # Check for text duplication
+        words = corrected.split()
+        has_duplicate = any(i > 0 and words[i] == words[i-1] for i in range(len(words)))
+
+        # Check for dropped words (output should have ≈ same word count ±1)
+        input_words = case["input"].split()
+        word_diff = len(words) - len(input_words)
+
+        result = {
+            "input": case["input"],
+            "corrected": corrected,
+            "suggestions": len(suggestions),
+            "has_duplicate": has_duplicate,
+            "word_count_diff": word_diff,
+        }
+        results.append(result)
+
+        print(f"\n  {case['desc']}")
+        print(f"    Input:     '{case['input']}'")
+        print(f"    Corrected: '{corrected}'")
+        print(f"    Suggestions: {len(suggestions)}")
+        if has_duplicate:
+            print(f"    ❌ DUPLICATE WORDS detected in output!")
+        else:
+            print(f"    ✅ No duplicate words")
+        if abs(word_diff) > 2:
+            print(f"    ⚠ Word count diff: {word_diff} (possible drop/duplication)")
+        else:
+            print(f"    ✅ Word count reasonable (diff={word_diff})")
+
+        for s in suggestions:
+            print(f"      [{s.get('start')}:{s.get('end')}] {s.get('type')}: '{s.get('original','')}' → '{s.get('correction','')}'")
+
+    return results
+
+
+# ══════════════════════════════════════════════════════════════════════
+# Phase 6.3 — BUG-017 Re-test (Intermittent Empty Response)
+# ══════════════════════════════════════════════════════════════════════
+def test_bug017():
+    print("\n" + "=" * 70)
+    print("PHASE 6.3 — BUG-017 Re-test (Intermittent Empty Response)")
+    print("=" * 70)
+
+    # Send the same input 5 times rapidly and check for empty responses
+    test_input = "الحديقه جميله والأزهار متفتحه"
+    empty_count = 0
+    error_count = 0
+    results = []
+
+    for i in range(5):
+        r = api_call("/api/analyze", test_input, timeout=30)
+        corrected = r.get("corrected", "")
+        suggestions = r.get("suggestions", [])
+        status = r.get("status", "")
+        warnings = r.get("warnings", {})
+
+        is_empty = (corrected == test_input and len(suggestions) == 0)
+        is_error = "error" in r and "status" not in r
+
+        if is_empty:
+            empty_count += 1
+        if is_error:
+            error_count += 1
+
+        result = {
+            "attempt": i + 1,
+            "corrected": corrected,
+            "suggestions": len(suggestions),
+            "status": status,
+            "warnings": warnings,
+            "is_empty": is_empty,
+            "is_error": is_error,
+        }
+        results.append(result)
+
+        status_str = "❌ EMPTY" if is_empty else ("❌ ERROR" if is_error else "✅ OK")
+        print(f"  Attempt {i+1}: {status_str} — suggestions={len(suggestions)}, status='{status}'")
+        if warnings:
+            print(f"    Warnings: {warnings}")
+        if is_error:
+            print(f"    Error: {r.get('error', '?')}")
+
+    print(f"\n  Empty responses: {empty_count}/5")
+    print(f"  Error responses: {error_count}/5")
+    if empty_count > 0:
+        print(f"  ⚠ BUG-017 may still be present!")
+    else:
+        print(f"  ✅ No empty responses detected")
+
+    return {
+        "empty_count": empty_count,
+        "error_count": error_count,
+        "results": results,
+    }
+
+
+# ══════════════════════════════════════════════════════════════════════
+# Phase 6.4 — 187-word Long Input Regression
+# ══════════════════════════════════════════════════════════════════════
+def test_long_input_regression():
+    print("\n" + "=" * 70)
+    print("PHASE 6.4 — 187-word Long Input Regression")
+    print("=" * 70)
+
+    long_text = (
+        "في ظل التطورات التكنولوجية المتسارعة التي يشهدها العالم اليوم أصبح من الضروري "
+        "أن نواكب هذه التغييرات ونتكيف معها بشكل فعال حيث تلعب التكنولوجيا دورا محوريا "
+        "في مختلف جوانب حياتنا اليومية بدءا من التعليم والصحة وصولا إلى الاقتصاد والسياسة "
+        "ولقد أدى الذكاء الاصطناعي إلى تحولات جذرية في طريقة عمل المؤسسات والشركات حيث "
+        "باتت الآلات قادرة على أداء مهام كانت حكرا على البشر مما يطرح تساؤلات عديدة حول "
+        "مستقبل سوق العمل والوظائف التقليدية كما أن التحول الرقمي فرض على الحكومات والمجتمعات "
+        "إعادة النظر في سياساتها التعليمية والاقتصادية لضمان مواكبة هذا التطور السريع وفي هذا "
+        "السياق يبرز دور البحث العلمي والابتكار كعاملين أساسيين في دفع عجلة التنمية المستدامة "
+        "وتحقيق الرفاهية للمجتمعات البشرية إذ لا يمكن لأي دولة أن تحقق تقدما حقيقيا دون "
+        "الاستثمار في العقول البشرية وتوفير بيئة محفزة للإبداع والابتكار ومن هنا تأتي أهمية "
+        "التعاون الدولي في مجال البحث العلمي وتبادل الخبرات والمعارف بين الدول المتقدمة والنامية "
+        "على حد سواء لتحقيق التنمية الشاملة والمستدامة التي تعود بالنفع على جميع شعوب العالم"
+    )
+    print(f"  Input: {len(long_text)} chars, {len(long_text.split())} words")
+
+    r = api_call("/api/analyze", long_text, timeout=120)
+    status = r.get("status", "")
+    corrected = r.get("corrected", "")
+    suggestions = r.get("suggestions", [])
+    warnings = r.get("warnings", {})
+    timing = r.get("timing_ms", {})
+
+    if "error" in r and "status" not in r:
+        print(f"  ❌ ERROR: {r['error']}")
+        result_status = "error"
+    elif status == "partial":
+        print(f"  ⚠ PARTIAL: some stages failed")
+        print(f"    Warnings: {warnings}")
+        result_status = "partial"
+    elif status == "success":
+        print(f"  ✅ SUCCESS")
+        result_status = "success"
+    else:
+        print(f"  ⚠ UNKNOWN STATUS: '{status}'")
+        result_status = "unknown"
+
+    print(f"  Elapsed: {r.get('_elapsed_ms', '?')}ms")
+    print(f"  Timing: {timing}")
+    print(f"  Suggestions: {len(suggestions)}")
+    print(f"  Corrected == Original: {corrected == long_text}")
+
+    # Key check: response should NOT be silently empty
+    is_silently_empty = (status == "success" and corrected == long_text and len(suggestions) == 0)
+    if is_silently_empty:
+        print(f"  ⚠ Silently empty! This is the BUG-032 behavior we're preventing.")
+    else:
+        print(f"  ✅ Response is either successful with results or properly flagged as partial/error")
+
+    return {
+        "input_chars": len(long_text),
+        "input_words": len(long_text.split()),
+        "status": result_status,
+        "suggestions": len(suggestions),
+        "warnings": warnings,
+        "timing": timing,
+        "elapsed_ms": r.get("_elapsed_ms"),
+        "is_silently_empty": is_silently_empty,
+    }
+
+
+# ══════════════════════════════════════════════════════════════════════
+# Phase 7.1 — BUG-018 Precise Tracing
+# ══════════════════════════════════════════════════════════════════════
+def test_bug018_tracing():
+    print("\n" + "=" * 70)
+    print("PHASE 7.1 — BUG-018 Precise Tracing (dropped ؛)")
+    print("=" * 70)
+
+    test_input = "قال المعلم للطلاب ادرسوا جيدا فالامتحان قريب"
+    print(f"  Input: '{test_input}'")
+
+    # Track A: Raw punctuation only
+    a = api_call("/api/punctuation", test_input)
+    a_out = a.get("corrected_text", test_input)
+    has_semicolon_raw = "؛" in a_out
+    print(f"\n  Raw punctuation output: '{a_out}'")
+    print(f"  Has ؛: {has_semicolon_raw}")
+
+    # Track B: Full pipeline
+    b = api_call("/api/analyze", test_input)
+    b_out = b.get("corrected", test_input)
+    b_sugg = b.get("suggestions", [])
+    has_semicolon_pipeline = "؛" in b_out
+    print(f"\n  Pipeline output: '{b_out}'")
+    print(f"  Has ؛: {has_semicolon_pipeline}")
+    print(f"  Suggestions: {len(b_sugg)}")
+
+    for s in b_sugg:
+        print(f"    [{s.get('start')}:{s.get('end')}] {s.get('type')}: '{s.get('original','')}' → '{s.get('correction','')}'")
+
+    # Determine drop cause
+    if has_semicolon_raw and not has_semicolon_pipeline:
+        # Raw produced it but pipeline dropped it
+        punc_suggestions = [s for s in b_sugg if s.get('type') == 'punctuation']
+        total_punc = len(punc_suggestions)
+        if total_punc >= 3:
+            cause = "MAX_PUNC_PATCHES_PER_RESPONSE cap (3 patches, ؛ was 4th+)"
+        else:
+            # Check if any grammar suggestion overlaps the ؛ position
+            cause = "StageLocker or validate_punctuation_diff rejection"
+        print(f"\n  DIAGNOSIS: ؛ was produced by raw model but dropped by pipeline")
+        print(f"  Likely cause: {cause}")
+    elif not has_semicolon_raw:
+        cause = "Raw punctuation model did NOT produce ؛ at all"
+        print(f"\n  DIAGNOSIS: {cause} — not a pipeline bug")
+    else:
+        cause = "؛ present in both raw and pipeline — BUG-018 not reproduced"
+        print(f"\n  DIAGNOSIS: {cause}")
+
+    return {
+        "input": test_input,
+        "raw_output": a_out,
+        "pipeline_output": b_out,
+        "has_semicolon_raw": has_semicolon_raw,
+        "has_semicolon_pipeline": has_semicolon_pipeline,
+        "diagnosis": cause,
+        "pipeline_punc_count": len([s for s in b_sugg if s.get('type') == 'punctuation']),
+    }
+
+
+# ══════════════════════════════════════════════════════════════════════
+# MAIN
+# ══════════════════════════════════════════════════════════════════════
+if __name__ == "__main__":
+    print("BAYAN — Gap-filling Live Tests\n")
+
+    all_results["phase_1_3"] = test_category9_live()
+    all_results["phase_2"] = test_rare_vocabulary()
+    all_results["phase_3_2"] = test_word_splits()
+    all_results["phase_5_5"] = test_dual_corrections()
+    all_results["phase_6_3"] = test_bug017()
+    all_results["phase_6_4"] = test_long_input_regression()
+    all_results["phase_7_1"] = test_bug018_tracing()
+
+    # Save all results
+    output_path = os.path.join(os.path.dirname(__file__), 'gap_filling_results.json')
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(all_results, f, ensure_ascii=False, indent=2)
+    print(f"\n\nAll results saved to {output_path}")

tests/phase0_investigation.py

@@ -0,0 +1,221 @@
+"""
+Phase 0 — Investigation Script
+Tests:
+  0.1 — ان→أن in sentence context vs isolation
+  0.3 — BUG-032 (long text) with detailed error capture
+  0.4 — BUG-031 sentence (already resolved: الطلاب = plural → اللذين is wrong)
+"""
+import sys, os, json, time, requests
+
+API_BASE = "https://bayan10-bayan-api.hf.space"
+TIMEOUT = 90
+
+def api_call(endpoint, text):
+    url = f"{API_BASE}{endpoint}"
+    try:
+        t0 = time.time()
+        resp = requests.post(url, json={"text": text}, timeout=TIMEOUT)
+        elapsed = int((time.time() - t0) * 1000)
+        if resp.status_code == 200:
+            data = resp.json()
+            data['_elapsed_ms'] = elapsed
+            return data
+        return {"error": f"HTTP {resp.status_code}", "_elapsed_ms": elapsed}
+    except Exception as e:
+        return {"error": f"{type(e).__name__}: {e}"}
+
+def test_0_1():
+    """0.1 — Does spelling correct ان→أن in sentence context?"""
+    print("=" * 70)
+    print("PHASE 0.1 — ان→أن contradiction test")
+    print("=" * 70)
+
+    tests = [
+        ("ان (isolation)", "ان"),
+        ("ان الحياة جميلة (sentence)", "ان الحياة جميلة"),
+        ("ان الذكاء مهم (sentence)", "ان الذكاء مهم"),
+        ("قال ان الحق واضح (mid-sentence)", "قال ان الحق واضح"),
+    ]
+
+    results = []
+    for label, text in tests:
+        # Track A: raw spelling model
+        a = api_call("/api/spelling", text)
+        a_out = a.get("corrected_text", text)
+        a_changed = a_out != text
+
+        # Track B: full pipeline
+        b = api_call("/api/analyze", text)
+        b_out = b.get("corrected", text)
+        b_sugg = b.get("suggestions", [])
+
+        result = {
+            "label": label, "input": text,
+            "raw_spelling": a_out, "raw_changed": a_changed,
+            "pipeline_corrected": b_out,
+            "pipeline_suggestions": len(b_sugg),
+        }
+        results.append(result)
+
+        print(f"\n  {label}:")
+        print(f"    Input:      '{text}'")
+        print(f"    Raw spell:  '{a_out}' (changed={a_changed})")
+        print(f"    Pipeline:   '{b_out}' (suggestions={len(b_sugg)})")
+
+        # Check if ان was corrected to أن or إن
+        if 'أن' in a_out or 'إن' in a_out:
+            print(f"    ✅ Raw spelling DID correct ان")
+        elif a_changed:
+            print(f"    ⚠ Raw spelling changed but NOT to أن/إن")
+        else:
+            print(f"    ❌ Raw spelling did NOT correct ان")
+
+    # Verdict
+    print("\n" + "-" * 50)
+    isolation = results[0]
+    sentences = results[1:]
+    iso_fixed = 'أن' in isolation['raw_spelling'] or 'إن' in isolation['raw_spelling']
+    sent_fixed = any('أن' in r['raw_spelling'] or 'إن' in r['raw_spelling'] for r in sentences)
+
+    if iso_fixed and sent_fixed:
+        verdict = "WORKS in both isolation AND sentence context"
+    elif iso_fixed and not sent_fixed:
+        verdict = "WORKS in isolation ONLY, FAILS in sentence context"
+    elif not iso_fixed:
+        verdict = "FAILS in both isolation and sentence context"
+    else:
+        verdict = "Inconsistent"
+
+    print(f"  FINAL VERDICT: {verdict}")
+    return {"verdict": verdict, "results": results}
+
+
+def test_0_3():
+    """0.3 — BUG-032: Long text (187 words / 1104 chars)"""
+    print("\n" + "=" * 70)
+    print("PHASE 0.3 — BUG-032 long text test")
+    print("=" * 70)
+
+    # 187-word Arabic text (from deep-dive report)
+    long_text = (
+        "في ظل التطورات التكنولوجية المتسارعة التي يشهدها العالم اليوم أصبح من الضروري "
+        "أن نواكب هذه التغييرات ونتكيف معها بشكل فعال حيث تلعب التكنولوجيا دورا محوريا "
+        "في مختلف جوانب حياتنا اليومية بدءا من التعليم والصحة وصولا إلى الاقتصاد والسياسة "
+        "ولقد أدى الذكاء الاصطناعي إلى تحولات جذرية في طريقة عمل المؤسسات والشركات حيث "
+        "باتت الآلات قادرة على أداء مهام كانت حكرا على البشر مما يطرح تساؤلات عديدة حول "
+        "مستقبل سوق العمل والوظائف التقليدية كما أن التحول الرقمي فرض على الحكومات والمجتمعات "
+        "إعادة النظر في سياساتها التعليمية والاقتصادية لضمان مواكبة هذا التطور السريع وفي هذا "
+        "السياق يبرز دور البحث العلمي والابتكار كعاملين أساسيين في دفع عجلة التنمية المستدامة "
+        "وتحقيق الرفاهية للمجتمعات البشرية إذ لا يمكن لأي دولة أن تحقق تقدما حقيقيا دون "
+        "الاستثمار في العقول ��لبشرية وتوفير بيئة محفزة للإبداع والابتكار ومن هنا تأتي أهمية "
+        "التعاون الدولي في مجال البحث العلمي وتبادل الخبرات والمعارف بين الدول المتقدمة والنامية "
+        "على حد سواء لتحقيق التنمية الشاملة والمستدامة التي تعود بالنفع على جميع شعوب العالم"
+    )
+    print(f"  Input length: {len(long_text)} chars, {len(long_text.split())} words")
+
+    # Test all three individual endpoints
+    print("\n  Testing /api/spelling...")
+    a_spell = api_call("/api/spelling", long_text)
+    print(f"    Status: {'error' if 'error' in a_spell else 'OK'}")
+    if 'error' in a_spell:
+        print(f"    Error: {a_spell['error']}")
+    else:
+        print(f"    Elapsed: {a_spell.get('_elapsed_ms', '?')}ms")
+        print(f"    Changed: {a_spell.get('corrected_text', '') != long_text}")
+
+    print("\n  Testing /api/grammar...")
+    a_gram = api_call("/api/grammar", long_text)
+    print(f"    Status: {'error' if 'error' in a_gram else 'OK'}")
+    if 'error' in a_gram:
+        print(f"    Error: {a_gram['error']}")
+    else:
+        print(f"    Elapsed: {a_gram.get('_elapsed_ms', '?')}ms")
+        print(f"    Changed: {a_gram.get('corrected_text', '') != long_text}")
+
+    print("\n  Testing /api/punctuation...")
+    a_punc = api_call("/api/punctuation", long_text)
+    print(f"    Status: {'error' if 'error' in a_punc else 'OK'}")
+    if 'error' in a_punc:
+        print(f"    Error: {a_punc['error']}")
+    else:
+        print(f"    Elapsed: {a_punc.get('_elapsed_ms', '?')}ms")
+        print(f"    Changed: {a_punc.get('corrected_text', '') != long_text}")
+
+    print("\n  Testing /api/analyze (full pipeline)...")
+    b = api_call("/api/analyze", long_text)
+    print(f"    Status: {'error' if 'error' in b and 'status' not in b else b.get('status', '?')}")
+    if 'error' in b and 'status' not in b:
+        print(f"    Error: {b['error']}")
+    else:
+        print(f"    Elapsed: {b.get('_elapsed_ms', '?')}ms")
+        print(f"    Suggestions: {len(b.get('suggestions', []))}")
+        print(f"    Timing: {b.get('timing_ms', {})}")
+        if b.get('corrected') == long_text:
+            print(f"    ⚠ corrected == original (no changes or silent failure?)")
+
+    return {
+        "input_chars": len(long_text),
+        "input_words": len(long_text.split()),
+        "spelling": {"error": a_spell.get("error"), "elapsed": a_spell.get("_elapsed_ms")},
+        "grammar": {"error": a_gram.get("error"), "elapsed": a_gram.get("_elapsed_ms")},
+        "punctuation": {"error": a_punc.get("error"), "elapsed": a_punc.get("_elapsed_ms")},
+        "pipeline": {
+            "error": b.get("error"),
+            "status": b.get("status"),
+            "suggestions": len(b.get("suggestions", [])),
+            "timing": b.get("timing_ms", {}),
+            "elapsed": b.get("_elapsed_ms"),
+        }
+    }
+
+
+def test_0_4():
+    """0.4 — BUG-031: اللذين vs الذين"""
+    print("\n" + "=" * 70)
+    print("PHASE 0.4 — BUG-031 (اللذين vs الذين)")
+    print("=" * 70)
+
+    sentence = "الطلاب اللذين اجتهدو في دراستهم حققو نتائج ممتازه في الأمتحانات الصعبه"
+    print(f"  Test sentence: '{sentence}'")
+    print(f"  Subject: الطلاب (PLURAL, not dual)")
+    print(f"  Therefore: اللذين (dual) is WRONG, الذين (plural) is CORRECT")
+    print(f"  Verdict: BUG-031 IS a real bug — grammar should correct اللذين→الذين")
+
+    # Test it
+    a_gram = api_call("/api/grammar", sentence)
+    a_out = a_gram.get("corrected_text", sentence)
+    print(f"\n  Grammar model output: '{a_out}'")
+    if 'الذين' in a_out and 'اللذين' not in a_out:
+        print(f"  ✅ Grammar DID correct اللذين→الذين")
+        bug_status = "fixed_by_model"
+    elif 'اللذين' in a_out:
+        print(f"  ❌ Grammar did NOT correct اللذين (left as dual)")
+        bug_status = "still_broken"
+    else:
+        print(f"  ⚠ Unexpected output")
+        bug_status = "unclear"
+
+    return {
+        "sentence": sentence,
+        "subject": "الطلاب (PLURAL)",
+        "correct_form": "الذين (plural)",
+        "is_real_bug": True,
+        "grammar_output": a_out,
+        "bug_status": bug_status,
+    }
+
+
+if __name__ == "__main__":
+    print("BAYAN Phase 0 — Investigation\n")
+
+    all_results = {}
+
+    all_results["phase_0_1"] = test_0_1()
+    all_results["phase_0_3"] = test_0_3()
+    all_results["phase_0_4"] = test_0_4()
+
+    # Save results
+    output_path = os.path.join(os.path.dirname(__file__), 'phase0_results.json')
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(all_results, f, ensure_ascii=False, indent=2)
+    print(f"\nResults saved to {output_path}")

tests/phase0_results.json

@@ -0,0 +1,75 @@
+{
+  "phase_0_1": {
+    "verdict": "WORKS in isolation ONLY, FAILS in sentence context",
+    "results": [
+      {
+        "label": "ان (isolation)",
+        "input": "ان",
+        "raw_spelling": "أن",
+        "raw_changed": true,
+        "pipeline_corrected": "إن.",
+        "pipeline_suggestions": 1
+      },
+      {
+        "label": "ان الحياة جميلة (sentence)",
+        "input": "ان الحياة جميلة",
+        "raw_spelling": "ان الحياة جميلة",
+        "raw_changed": false,
+        "pipeline_corrected": "إن الحياة جميلة!",
+        "pipeline_suggestions": 2
+      },
+      {
+        "label": "ان الذكاء مهم (sentence)",
+        "input": "ان الذكاء مهم",
+        "raw_spelling": "ان الذكاء مهم",
+        "raw_changed": false,
+        "pipeline_corrected": "إن الذكاء مهم.",
+        "pipeline_suggestions": 2
+      },
+      {
+        "label": "قال ان الحق واضح (mid-sentence)",
+        "input": "قال ان الحق واضح",
+        "raw_spelling": "قال ان الحق واضح",
+        "raw_changed": false,
+        "pipeline_corrected": "قال ان: الحق واضح",
+        "pipeline_suggestions": 1
+      }
+    ]
+  },
+  "phase_0_3": {
+    "input_chars": 982,
+    "input_words": 159,
+    "spelling": {
+      "error": "ReadTimeout: HTTPSConnectionPool(host='bayan10-bayan-api.hf.space', port=443): Read timed out. (read timeout=90)",
+      "elapsed": null
+    },
+    "grammar": {
+      "error": "ReadTimeout: HTTPSConnectionPool(host='bayan10-bayan-api.hf.space', port=443): Read timed out. (read timeout=90)",
+      "elapsed": null
+    },
+    "punctuation": {
+      "error": "ReadTimeout: HTTPSConnectionPool(host='bayan10-bayan-api.hf.space', port=443): Read timed out. (read timeout=90)",
+      "elapsed": null
+    },
+    "pipeline": {
+      "error": null,
+      "status": "success",
+      "suggestions": 4,
+      "timing": {
+        "grammar_ms": 12179,
+        "punctuation_ms": 12237,
+        "spelling_ms": 0,
+        "total_ms": 24420
+      },
+      "elapsed": 54892
+    }
+  },
+  "phase_0_4": {
+    "sentence": "الطلاب اللذين اجتهدو في دراستهم حققو نتائج ممتازه في الأمتحانات الصعبه",
+    "subject": "الطلاب (PLURAL)",
+    "correct_form": "الذين (plural)",
+    "is_real_bug": true,
+    "grammar_output": "الطلاب الذين اجتهدو في دراستهم حققوا نتائج ممتازة في الامتحانات الصعبة",
+    "bug_status": "fixed_by_model"
+  }
+}

tests/phase5_investigation.py

@@ -0,0 +1,161 @@
+"""
+Phase 5 — BUG-016/027 Text Duplication Investigation
+
+Reproduce exact case: ان الطالبات ذهبو الى الجامعه
+Log every patch produced by spelling and grammar with full ORIGINAL coordinates.
+Determine: overlapping coords (PatchSet bug) vs non-overlapping (coord computation bug).
+Also check: does الى get silently dropped?
+"""
+import sys, os, json, time, requests
+
+API_BASE = "https://bayan10-bayan-api.hf.space"
+TIMEOUT = 60
+
+def api_call(endpoint, text):
+    url = f"{API_BASE}{endpoint}"
+    try:
+        t0 = time.time()
+        resp = requests.post(url, json={"text": text}, timeout=TIMEOUT)
+        elapsed = int((time.time() - t0) * 1000)
+        if resp.status_code == 200:
+            data = resp.json()
+            data['_elapsed_ms'] = elapsed
+            return data
+        return {"error": f"HTTP {resp.status_code}", "_elapsed_ms": elapsed}
+    except Exception as e:
+        return {"error": f"{type(e).__name__}: {e}"}
+
+
+def investigate_bug_016():
+    """Full coordinate investigation for BUG-016."""
+    print("=" * 70)
+    print("PHASE 5 — BUG-016/027 Text Duplication Investigation")
+    print("=" * 70)
+
+    test_input = "ان الطالبات ذهبو الى الجامعه"
+    print(f"\nInput: '{test_input}'")
+    print(f"Words: {test_input.split()}")
+    for i, w in enumerate(test_input.split()):
+        # Compute char offsets
+        start = test_input.index(w) if i == 0 else test_input.index(w, sum(len(x) + 1 for x in test_input.split()[:i]))
+        end = start + len(w)
+        print(f"  Word {i}: '{w}' chars [{start}:{end}]")
+
+    # Track A: Raw model outputs
+    print("\n--- Track A: Raw Spelling ---")
+    a_spell = api_call("/api/spelling", test_input)
+    a_spell_out = a_spell.get("corrected_text", test_input)
+    print(f"  Input:  '{test_input}'")
+    print(f"  Output: '{a_spell_out}'")
+    print(f"  Changed: {a_spell_out != test_input}")
+
+    # Character-level diff
+    if a_spell_out != test_input:
+        print("\n  Character-level changes (spelling):")
+        from difflib import SequenceMatcher
+        s = SequenceMatcher(None, test_input.split(), a_spell_out.split())
+        for tag, i1, i2, j1, j2 in s.get_opcodes():
+            if tag != 'equal':
+                orig_words = test_input.split()[i1:i2]
+                corr_words = a_spell_out.split()[j1:j2]
+                print(f"    {tag}: [{i1}:{i2}] {orig_words} → [{j1}:{j2}] {corr_words}")
+
+    print("\n--- Track A: Raw Grammar ---")
+    a_gram = api_call("/api/grammar", test_input)
+    a_gram_out = a_gram.get("corrected_text", test_input)
+    print(f"  Input:  '{test_input}'")
+    print(f"  Output: '{a_gram_out}'")
+    print(f"  Changed: {a_gram_out != test_input}")
+
+    if a_gram_out != test_input:
+        print("\n  Character-level changes (grammar):")
+        from difflib import SequenceMatcher
+        s = SequenceMatcher(None, test_input.split(), a_gram_out.split())
+        for tag, i1, i2, j1, j2 in s.get_opcodes():
+            if tag != 'equal':
+                orig_words = test_input.split()[i1:i2]
+                corr_words = a_gram_out.split()[j1:j2]
+                print(f"    {tag}: [{i1}:{i2}] {orig_words} → [{j1}:{j2}] {corr_words}")
+
+    # Track B: Full pipeline
+    print("\n--- Track B: Full Pipeline ---")
+    b = api_call("/api/analyze", test_input)
+    b_corrected = b.get("corrected", test_input)
+    b_suggestions = b.get("suggestions", [])
+    print(f"  Input:      '{test_input}'")
+    print(f"  Corrected:  '{b_corrected}'")
+    print(f"  Suggestions: {len(b_suggestions)}")
+
+    for s in b_suggestions:
+        print(f"\n    Suggestion [{s.get('start')}:{s.get('end')}]:")
+        print(f"      Type: {s.get('type')}")
+        print(f"      Original: '{s.get('original', '')}'")
+        print(f"      Correction: '{s.get('correction', '')}'")
+        if 'confidence' in s:
+            print(f"      Confidence: {s.get('confidence')}")
+
+    # Check for duplicates
+    print("\n--- Duplicate / Drop Analysis ---")
+    output_words = b_corrected.split()
+    input_words = test_input.split()
+    print(f"  Input words:  {input_words}")
+    print(f"  Output words: {output_words}")
+
+    # Check for duplicated words
+    for i, w in enumerate(output_words):
+        if i > 0 and w == output_words[i-1]:
+            print(f"  ⚠ DUPLICATE: '{w}' at positions {i-1} and {i}")
+
+    # Check for dropped words (الى should appear as الى or إلى)
+    for w in input_words:
+        # Check if word or a known correction of it appears in output
+        found = w in b_corrected
+        if not found:
+            # Check common corrections
+            corrections = {
+                'ان': ['أن', 'إن', 'ان'],
+                'الى': ['إلى', 'الى'],
+                'الجامعه': ['الجامعة', 'الجامعه'],
+                'ذهبو': ['ذهبوا', 'ذهبن', 'ذهبو'],
+                'الطالبات': ['الطالبات'],
+            }
+            alts = corrections.get(w, [w])
+            found = any(a in b_corrected for a in alts)
+        if not found:
+            print(f"  ⚠ DROPPED: '{w}' not found in corrected output!")
+        else:
+            print(f"  ✓ '{w}' present (or corrected variant)")
+
+    # Overlap analysis between suggestions
+    print("\n--- Overlap Analysis ---")
+    for i, s1 in enumerate(b_suggestions):
+        for j, s2 in enumerate(b_suggestions):
+            if j <= i:
+                continue
+            s1_start, s1_end = s1.get('start', 0), s1.get('end', 0)
+            s2_start, s2_end = s2.get('start', 0), s2.get('end', 0)
+            if s1_start < s2_end and s2_start < s1_end:
+                print(f"  ⚠ OVERLAP: suggestion {i} [{s1_start}:{s1_end}] and suggestion {j} [{s2_start}:{s2_end}]")
+                print(f"    S{i}: '{s1.get('original','')}' → '{s1.get('correction','')}' ({s1.get('type')})")
+                print(f"    S{j}: '{s2.get('original','')}' → '{s2.get('correction','')}' ({s2.get('type')})")
+    if not any(
+        s1.get('start', 0) < s2.get('end', 0) and s2.get('start', 0) < s1.get('end', 0)
+        for i, s1 in enumerate(b_suggestions) for j, s2 in enumerate(b_suggestions) if j > i
+    ):
+        print("  ✓ No overlapping suggestions found")
+
+    return {
+        "input": test_input,
+        "raw_spelling": a_spell_out,
+        "raw_grammar": a_gram_out,
+        "pipeline_corrected": b_corrected,
+        "suggestions": b_suggestions,
+    }
+
+
+if __name__ == "__main__":
+    result = investigate_bug_016()
+    output_path = os.path.join(os.path.dirname(__file__), 'phase5_results.json')
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+    print(f"\nResults saved to {output_path}")

tests/phase5_results.json

@@ -0,0 +1,61 @@
+{
+  "input": "ان الطالبات ذهبو الى الجامعه",
+  "raw_spelling": "ان الط ابت ذهبوا إلى الجامعه",
+  "raw_grammar": "إن الطالبات ذهبن إلى الجامعة",
+  "pipeline_corrected": "إن الطالبات ذهبن ذهبوا الجامعة.",
+  "suggestions": [
+    {
+      "alternatives": [],
+      "confidence": 1.0,
+      "correction": "ذهبن",
+      "end": 16,
+      "id": "f303a4d8-1369-43f7-8ad5-209c03d4af24",
+      "locked": true,
+      "original": "ذهبو",
+      "priority": 3,
+      "start": 12,
+      "type": "grammar"
+    },
+    {
+      "alternatives": [],
+      "confidence": 0.8,
+      "correction": "الجامعة.",
+      "end": 28,
+      "id": "ece1c300-e501-44dc-9ef2-907b47785145",
+      "locked": true,
+      "original": "الجامعه",
+      "priority": 2,
+      "start": 21,
+      "type": "punctuation"
+    },
+    {
+      "alternatives": [],
+      "confidence": 1.0,
+      "correction": "إن",
+      "end": 2,
+      "id": "aa123654-bb3a-46ab-aa3c-7cea6dc4955b",
+      "locked": true,
+      "original": "ان",
+      "priority": 1,
+      "start": 0,
+      "type": "spelling"
+    },
+    {
+      "alternatives": [
+        "ذهبوا",
+        "ال",
+        "الم",
+        "الى"
+      ],
+      "confidence": 0.9,
+      "correction": "ذهبوا",
+      "end": 20,
+      "id": "bf07637f-0432-4311-aab1-77f521718214",
+      "locked": true,
+      "original": "الى",
+      "priority": 1,
+      "start": 17,
+      "type": "spelling"
+    }
+  ]
+}

tests/phase9_results.json

@@ -0,0 +1,27 @@
+{
+  "test_count": 5,
+  "all_pass": true,
+  "contamination_found": false,
+  "results": {
+    "CONC-3": {
+      "corrected": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة؛ في بناء كتلة عضلية قوية ويا له، من التزام حديدي يثير الإعجاب ولقد كان! أداؤه في المسابقات الأخيرة مبهرا للغاية",
+      "suggestions_count": 3
+    },
+    "CONC-1": {
+      "corrected": "الحديقه جميلة.",
+      "suggestions_count": 1
+    },
+    "CONC-2": {
+      "corrected": "الطلاب ذهبوا إلى المدرسة.",
+      "suggestions_count": 3
+    },
+    "CONC-5": {
+      "corrected": "هذه المدينة جميلة جدا ومناخها معتدل.",
+      "suggestions_count": 2
+    },
+    "CONC-4": {
+      "corrected": "القاهرة عاصمة مصر.",
+      "suggestions_count": 1
+    }
+  }
+}

tests/round2_b2_b3_live.json

@@ -0,0 +1,37 @@
+{
+  "b2": [
+    {
+      "id": "BUG-006",
+      "word": "اهم",
+      "input": "هذا اهم شيء في الحياة",
+      "corrected": "هذا اهم شيء في الحياة.",
+      "status": "✅ PRESERVED",
+      "targeting_suggestions": 0
+    },
+    {
+      "id": "BUG-009",
+      "word": "قرأ",
+      "input": "قرأ الطالب الكتاب",
+      "corrected": "قرأ الطالب الكتاب.",
+      "status": "✅ PRESERVED",
+      "targeting_suggestions": 0
+    },
+    {
+      "id": "BUG-010",
+      "word": "مشى",
+      "input": "مشى الرجل إلى البيت",
+      "corrected": "مشى الرجل إلى البيت.",
+      "status": "✅ PRESERVED",
+      "targeting_suggestions": 0
+    },
+    {
+      "id": "BUG-013",
+      "word": "خطأ",
+      "input": "وقع في خطأ كبير",
+      "corrected": "وقع في خطأ كبير.",
+      "status": "✅ PRESERVED",
+      "targeting_suggestions": 0
+    }
+  ],
+  "b3_note": "Deployed API lacks Round 2 fixes"
+}

tests/round2_b2_b3_live.py

@@ -0,0 +1,102 @@
+"""
+Round 2 — B2 Live API Test
+Tests BUG-006/009/010/013 through the DEPLOYED pipeline to verify
+whether the existing mechanism catches common-word substitutions.
+"""
+import requests, json, time, os
+
+API_BASE = "https://bayan10-bayan-api.hf.space"
+TIMEOUT = 60
+
+def api_call(endpoint, text):
+    try:
+        t0 = time.time()
+        resp = requests.post(f"{API_BASE}{endpoint}", json={"text": text}, timeout=TIMEOUT)
+        elapsed = int((time.time() - t0) * 1000)
+        if resp.status_code == 200:
+            data = resp.json()
+            data['_elapsed_ms'] = elapsed
+            return data
+        return {"error": f"HTTP {resp.status_code}", "_elapsed_ms": elapsed}
+    except Exception as e:
+        return {"error": str(e)}
+
+print("=" * 70)
+print("B2 LIVE TEST: Common-word substitution via /api/analyze")
+print("=" * 70)
+
+# Test each BUG in sentence context
+tests = [
+    {"id": "BUG-006", "sentence": "هذا اهم شيء في الحياة", "word": "اهم",
+     "bad_correction": "مهم", "concern": "اهم must NOT become مهم"},
+    {"id": "BUG-009", "sentence": "قرأ الطالب الكتاب", "word": "قرأ",
+     "bad_correction": "قرا", "concern": "قرأ must NOT become قرا"},
+    {"id": "BUG-010", "sentence": "مشى الرجل إلى البيت", "word": "مشى",
+     "bad_correction": "مضى", "concern": "مشى must NOT become مضى"},
+    {"id": "BUG-013", "sentence": "وقع في خطأ كبير", "word": "خطأ",
+     "bad_correction": "خطا", "concern": "خطأ must NOT become خطا"},
+]
+
+results = []
+for t in tests:
+    r = api_call("/api/analyze", t["sentence"])
+    corrected = r.get("corrected", t["sentence"])
+    suggestions = r.get("suggestions", [])
+    
+    # Check if the target word was changed
+    word_present = t["word"] in corrected
+    bad_present = t["bad_correction"] in corrected and t["bad_correction"] not in t["sentence"]
+    
+    # Find suggestions targeting this word
+    targeting = [s for s in suggestions if t["word"] in s.get("original", "")
+                 or t["bad_correction"] in s.get("correction", "")]
+    
+    status = "❌ CORRUPTED" if bad_present else ("✅ PRESERVED" if word_present else "⚠ OTHER")
+    
+    result = {
+        "id": t["id"], "word": t["word"], "input": t["sentence"],
+        "corrected": corrected, "status": status,
+        "targeting_suggestions": len(targeting),
+    }
+    results.append(result)
+    
+    print(f"\n  {t['id']}: {t['concern']}")
+    print(f"    Input:     '{t['sentence']}'")
+    print(f"    Corrected: '{corrected}'")
+    print(f"    Status: {status}")
+    if targeting:
+        for s in targeting:
+            conf = s.get('confidence', '?')
+            print(f"    Suggestion: '{s.get('original','')}' → '{s.get('correction','')}' (conf={conf})")
+
+# Also test BUG-014/015 live
+print("\n" + "=" * 70)
+print("B3 LIVE TEST: Suffix corruption via /api/analyze")
+print("=" * 70)
+
+b3_tests = [
+    {"id": "BUG-014", "sentence": "قرأته بسرعة", "word": "قرأته",
+     "bad": "قرأتة", "concern": "قرأته must NOT become قرأتة"},
+    {"id": "BUG-015", "sentence": "استوقفني المشهد فتأملته مليا", "word": "فتأملته",
+     "bad": "فتأملتة", "concern": "فتأملته must NOT become فتأملتة"},
+]
+
+for t in b3_tests:
+    r = api_call("/api/analyze", t["sentence"])
+    corrected = r.get("corrected", t["sentence"])
+    bad_present = t["bad"] in corrected
+    word_present = t["word"] in corrected
+    status = "❌ CORRUPTED" if bad_present else ("✅ PRESERVED" if word_present else "⚠ OTHER")
+    
+    print(f"\n  {t['id']}: {t['concern']}")
+    print(f"    Input:     '{t['sentence']}'")
+    print(f"    Corrected: '{corrected}'")
+    print(f"    Status: {status}")
+    print(f"    NOTE: Deployed API does NOT have Round 2 fixes yet. "
+          f"This tests the CURRENT deployed state.")
+
+# Save
+output_path = os.path.join(os.path.dirname(__file__), 'round2_b2_b3_live.json')
+with open(output_path, 'w', encoding='utf-8') as f:
+    json.dump({"b2": results, "b3_note": "Deployed API lacks Round 2 fixes"}, f, ensure_ascii=False, indent=2)
+print(f"\nResults saved to {output_path}")

tests/round2_live_tests.py

@@ -0,0 +1,423 @@
+"""
+BAYAN Round 2 — Live API Tests
+Covers: A1 (BUG-032 exact input), A2 (Appendix E FP rate), B1 (31 prefix cases),
+        B4 (300/301 boundary), B5 (shadda), B7 (brackets)
+"""
+import sys, os, json, time, requests
+from datetime import datetime, timezone
+
+API_BASE = "https://bayan10-bayan-api.hf.space"
+TIMEOUT = 120
+
+def api_call(endpoint, text, timeout=TIMEOUT):
+    url = f"{API_BASE}{endpoint}"
+    try:
+        t0 = time.time()
+        resp = requests.post(url, json={"text": text}, timeout=timeout)
+        elapsed = int((time.time() - t0) * 1000)
+        if resp.status_code == 200:
+            data = resp.json()
+            data['_elapsed_ms'] = elapsed
+            return data
+        return {"error": f"HTTP {resp.status_code}", "_elapsed_ms": elapsed}
+    except Exception as e:
+        return {"error": f"{type(e).__name__}: {e}"}
+
+def log(msg):
+    print(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}", flush=True)
+
+results = {"timestamp": datetime.now(timezone.utc).isoformat()}
+
+# ═══════════════════════════════════════════════════════════════
+# A1: BUG-032 — Exact 1104-char/187-word Input Re-test
+# ═══════════════════════════════════════════════════════════════
+log("=" * 70)
+log("A1: BUG-032 — Exact Original Input Re-test")
+log("=" * 70)
+
+# Exact original text from phase0_investigation.py L100-112
+BUG032_TEXT = (
+    "في ظل التطورات التكنولوجية المتسارعة التي يشهدها العالم اليوم أصبح من الضروري "
+    "أن نواكب هذه التغييرات ونتكيف معها بشكل فعال حيث تلعب التكنولوجيا دورا محوريا "
+    "في مختلف جوانب حياتنا اليومية بدءا من التعليم والصحة وصولا إلى الاقتصاد والسياسة "
+    "ولقد أدى الذكاء الاصطناعي إلى تحولات جذرية في طريقة عمل المؤسسات والشركات حيث "
+    "باتت الآلات قادرة على أداء مهام كانت حكرا على البشر مما يطرح تساؤلات عديدة حول "
+    "مستقبل سوق العمل والوظائف التقليدية كما أن التحول الرقمي فرض على الحكومات والمجتمعات "
+    "إعادة النظر في سياساتها التعليمية والاقتصادية لضمان مواكبة هذا التطور السريع وفي هذا "
+    "السياق يبرز دور البحث العلمي والابتكار كعاملين أساسيين في دفع عجلة التنمية المستدامة "
+    "وتحقيق الرفاهية للمجتمعات البشرية إذ لا يمكن لأي دولة أن تحقق تقدما حقيقيا دون "
+    "الاستثمار في العقول البشرية وتوفير بيئة محفزة للإبداع والابتكار ومن هنا تأتي أهمية "
+    "التعاون الدولي في مجال البحث العلمي وتبادل الخبرات والمعارف بين الدول المتقدمة والنامية "
+    "على حد سواء لتحقيق التنمية الشاملة والمستدامة التي تعود بالنفع على جميع شعوب العالم"
+)
+
+log(f"  Input: {len(BUG032_TEXT)} chars, {len(BUG032_TEXT.split())} words")
+
+# Test 1: Full pipeline
+log("  Running /api/analyze...")
+a1_pipeline = api_call("/api/analyze", BUG032_TEXT)
+log(f"  Pipeline: status={a1_pipeline.get('status', 'N/A')}, "
+    f"suggestions={len(a1_pipeline.get('suggestions', []))}, "
+    f"elapsed={a1_pipeline.get('_elapsed_ms', 'N/A')}ms")
+if 'timing_ms' in a1_pipeline:
+    log(f"  Timing: {a1_pipeline['timing_ms']}")
+if 'warnings' in a1_pipeline:
+    log(f"  Warnings: {a1_pipeline['warnings']}")
+
+# Test 2: Individual endpoints for timing reconciliation
+log("  Running /api/spelling (expect timeout or skip)...")
+a1_spell = api_call("/api/spelling", BUG032_TEXT, timeout=120)
+log(f"  Spelling: elapsed={a1_spell.get('_elapsed_ms', 'N/A')}ms, "
+    f"error={a1_spell.get('error', 'none')}")
+
+log("  Running /api/grammar...")
+a1_gram = api_call("/api/grammar", BUG032_TEXT, timeout=120)
+log(f"  Grammar: elapsed={a1_gram.get('_elapsed_ms', 'N/A')}ms, "
+    f"changed={a1_gram.get('corrected_text', BUG032_TEXT) != BUG032_TEXT}")
+
+log("  Running /api/punctuation...")
+a1_punc = api_call("/api/punctuation", BUG032_TEXT, timeout=120)
+log(f"  Punctuation: elapsed={a1_punc.get('_elapsed_ms', 'N/A')}ms, "
+    f"changed={a1_punc.get('corrected_text', BUG032_TEXT) != BUG032_TEXT}")
+
+results['a1_bug032'] = {
+    'input_chars': len(BUG032_TEXT),
+    'input_words': len(BUG032_TEXT.split()),
+    'pipeline': {
+        'status': a1_pipeline.get('status'),
+        'suggestions': len(a1_pipeline.get('suggestions', [])),
+        'elapsed_ms': a1_pipeline.get('_elapsed_ms'),
+        'timing_ms': a1_pipeline.get('timing_ms'),
+        'warnings': a1_pipeline.get('warnings'),
+    },
+    'spelling': {'elapsed_ms': a1_spell.get('_elapsed_ms'), 'error': a1_spell.get('error')},
+    'grammar': {'elapsed_ms': a1_gram.get('_elapsed_ms')},
+    'punctuation': {'elapsed_ms': a1_punc.get('_elapsed_ms')},
+}
+
+# ═══════════════════════════════════════════════════════════════
+# A2: Appendix E FP Rate — EXACT Original Sentences
+# ═══════════════════════════════════════════════════════════════
+log("\n" + "=" * 70)
+log("A2: Appendix E — Exact Original Sentences FP Rate")
+log("=" * 70)
+
+# EXACT sentences from deep_dive_gaps.py L260-271
+APPENDIX_E_ORIGINAL = [
+    {"id": "R-01", "sentence": "استوقفني المشهد فتأملته مليا", "domain": "literary"},
+    {"id": "R-02", "sentence": "تستأثر القوى العظمى بالنفوذ الدولي", "domain": "political"},
+    {"id": "R-03", "sentence": "استقطب المؤتمر ثلة من العلماء الأفذاذ", "domain": "formal"},
+    {"id": "R-04", "sentence": "يتسنى للمرء أن يستشف الحقيقة من بين السطور", "domain": "literary_verb"},
+    {"id": "R-05", "sentence": "ألقى المحاضر خطبة عصماء استحوذت على إعجاب الحاضرين", "domain": "oratory"},
+    {"id": "R-06", "sentence": "تمخض الاجتماع عن قرارات مصيرية", "domain": "formal_verb"},
+    {"id": "R-07", "sentence": "أرهقته المسغبة فاستكان للقدر", "domain": "classical"},
+    {"id": "R-08", "sentence": "نستشرف آفاق المستقبل بثقة واقتدار", "domain": "formal_speech"},
+    {"id": "R-09", "sentence": "اعتراه القلق فتملكه الأرق", "domain": "literary_psych"},
+    {"id": "R-10", "sentence": "استأنف العمل بعد فترة من التقاعس", "domain": "formal_verb"},
+]
+
+# Constructed sentences from gap_filling_tests.py (for comparison)
+CONSTRUCTED_SENTENCES = [
+    {"id": "R-01c", "sentence": "المدينة العصماء تحتضن آلاف السكان", "word": "العصماء"},
+    {"id": "R-02c", "sentence": "يستشف الباحث نتائج الدراسة بعناية", "word": "يستشف"},
+    {"id": "R-03c", "sentence": "أرهقته المسغبة والعطش الشديد", "word": "المسغبة"},
+    {"id": "R-04c", "sentence": "التقاعس عن العمل يؤدي إلى الفشل", "word": "التقاعس"},
+    {"id": "R-05c", "sentence": "استئثار السلطة يهدد الديمقراطية", "word": "استئثار"},
+    {"id": "R-06c", "sentence": "تبجيل العلماء واجب على المجتمع", "word": "تبجيل"},
+    {"id": "R-07c", "sentence": "الرجل الدمث يحبه الجميع", "word": "الدمث"},
+    {"id": "R-08c", "sentence": "استقصاء الحقائق مهم في الصحافة", "word": "استقصاء"},
+    {"id": "R-09c", "sentence": "لا يجوز التواني في طلب العلم", "word": "التواني"},
+    {"id": "R-10c", "sentence": "كتاب المستطرف من أمهات الكتب العربية", "word": "المستطرف"},
+]
+
+def test_fp_set(name, items):
+    fp_count = 0
+    item_results = []
+    for item in items:
+        text = item["sentence"]
+        r = api_call("/api/analyze", text)
+        corrected = r.get("corrected", text)
+        suggestions = r.get("suggestions", [])
+        changed = corrected != text
+        if changed:
+            fp_count += 1
+        result = {
+            "id": item["id"], "input": text, "corrected": corrected,
+            "changed": changed, "suggestion_count": len(suggestions),
+        }
+        item_results.append(result)
+        status = "❌ CHANGED" if changed else "✅ PRESERVED"
+        log(f"  {item['id']}: {status}")
+        if changed:
+            log(f"    Input:     '{text[:60]}'")
+            log(f"    Corrected: '{corrected[:60]}'")
+            for s in suggestions:
+                log(f"      [{s.get('type','')}] '{s.get('original','')}' → '{s.get('correction','')}'")
+    log(f"\n  {name} FP rate: {fp_count}/{len(items)} = {fp_count*100//len(items)}%")
+    return {"fp_count": fp_count, "total": len(items), "fp_rate": f"{fp_count*100//len(items)}%", "results": item_results}
+
+log("\n  --- Original Appendix E sentences ---")
+a2_original = test_fp_set("Original Appendix E", APPENDIX_E_ORIGINAL)
+
+log("\n  --- Constructed sentences (comparison) ---")
+a2_constructed = test_fp_set("Constructed", CONSTRUCTED_SENTENCES)
+
+results['a2_fp_rate'] = {
+    'original_appendix_e': a2_original,
+    'constructed_sentences': a2_constructed,
+}
+
+# ═══════════════════════════════════════════════════════════════
+# B1: All 31 Prefix/Clitic Cases through Pipeline
+# ═══════════════════════════════════════════════════════════════
+log("\n" + "=" * 70)
+log("B1: All 31 Prefix/Clitic Cases through Pipeline")
+log("=" * 70)
+
+CAT8_ROOTS = ['مدرسة', 'شمس', 'أمة', 'نافذة', 'علم', 'اقتصاد']
+CAT8_PREFIXES = [("bare", ""), ("wa", "و"), ("ba", "ب"), ("la", "ل"), ("ka", "ك")]
+
+b1_total = 0
+b1_blocked = 0
+b1_leaked = 0
+b1_unchanged = 0
+b1_results = []
+
+for root in CAT8_ROOTS:
+    for pfx_name, pfx in CAT8_PREFIXES:
+        word = pfx + root
+        # Put word in a minimal sentence context
+        sentence = f"{word} مهم جدا"
+        r = api_call("/api/analyze", sentence)
+        corrected = r.get("corrected", sentence)
+        suggestions = r.get("suggestions", [])
+
+        # Check if the word was changed
+        word_in_corrected = word in corrected
+        word_changed = not word_in_corrected
+
+        # Find suggestions targeting this word
+        targeting = [s for s in suggestions if s.get("original", "").strip() == word
+                     or word in s.get("original", "")]
+
+        b1_total += 1
+        if not word_changed and not targeting:
+            b1_unchanged += 1
+            status = "✅ PRESERVED"
+        elif targeting and not word_changed:
+            # Suggestion exists but wasn't applied (dampened?)
+            conf = targeting[0].get('confidence', '?')
+            if conf and float(str(conf)) < 0.9:
+                b1_blocked += 1
+                status = f"✅ DAMPENED (conf={conf})"
+            else:
+                b1_leaked += 1
+                status = f"⚠ LEAKED (conf={conf})"
+        elif word_changed:
+            b1_leaked += 1
+            status = "❌ CHANGED"
+        else:
+            b1_unchanged += 1
+            status = "✅ OK"
+
+        result = {
+            "word": word, "root": root, "prefix": pfx_name,
+            "input": sentence, "corrected": corrected,
+            "word_preserved": word_in_corrected,
+            "targeting_suggestions": len(targeting),
+            "status": status,
+        }
+        b1_results.append(result)
+        log(f"  {word:12s} ({pfx_name:4s}+{root}): {status}")
+        if word_changed:
+            log(f"    Input:     '{sentence}'")
+            log(f"    Corrected: '{corrected}'")
+
+# BUG-021: ولذالك (case 31)
+sentence_31 = "ولذالك يجب الاهتمام"
+r31 = api_call("/api/analyze", sentence_31)
+corrected_31 = r31.get("corrected", sentence_31)
+word_31 = "ولذالك"
+word_31_ok = "ولذلك" in corrected_31
+bad_split_31 = "ولذا ذلك" in corrected_31
+b1_total += 1
+
+if word_31_ok:
+    status_31 = "✅ CORRECTED (ولذالك→ولذلك)"
+    b1_blocked += 1
+elif bad_split_31:
+    status_31 = "❌ BAD SPLIT (ولذا ذلك)"
+    b1_leaked += 1
+elif word_31 in corrected_31:
+    status_31 = "⚠ UNCHANGED (misspelling preserved)"
+    b1_unchanged += 1
+else:
+    status_31 = f"⚠ OTHER: '{corrected_31}'"
+    b1_leaked += 1
+
+b1_results.append({"word": word_31, "input": sentence_31, "corrected": corrected_31, "status": status_31})
+log(f"  {'ولذالك':12s} (BUG-021): {status_31}")
+
+log(f"\n  Total: {b1_total}, Preserved: {b1_unchanged}, Blocked/Dampened: {b1_blocked}, Leaked: {b1_leaked}")
+
+results['b1_prefix'] = {
+    'total': b1_total, 'unchanged': b1_unchanged,
+    'blocked': b1_blocked, 'leaked': b1_leaked,
+    'results': b1_results,
+}
+
+# ═══════════════════════════════════════════════════════════════
+# B4: 300/301 Boundary + Repetitive Text
+# ═══════════════════════════════════════════════════════════════
+log("\n" + "=" * 70)
+log("B4: 300/301 Boundary + Repetitive Text")
+log("=" * 70)
+
+# Test 1: Exact boundary (300 chars vs 301 chars)
+base = "ذهب الولد الى المدرسه وقابل المعلمه "  # ~37 chars with errors
+text_300 = (base * 10)[:300]
+text_301 = (base * 10)[:301]
+log(f"  300-char text: {len(text_300)} chars")
+log(f"  301-char text: {len(text_301)} chars")
+
+r300 = api_call("/api/analyze", text_300)
+r301 = api_call("/api/analyze", text_301)
+
+s300 = r300.get("suggestions", [])
+s301 = r301.get("suggestions", [])
+s300_types = {s.get('type') for s in s300}
+s301_types = {s.get('type') for s in s301}
+
+log(f"  300 chars: {len(s300)} suggestions, types={s300_types}")
+log(f"  301 chars: {len(s301)} suggestions, types={s301_types}")
+
+# Check if spelling suggestions differ
+s300_spell = [s for s in s300 if s.get('type') == 'spelling']
+s301_spell = [s for s in s301 if s.get('type') == 'spelling']
+log(f"  300 chars spelling: {len(s300_spell)} suggestions")
+log(f"  301 chars spelling: {len(s301_spell)} suggestions")
+
+if len(s300_spell) > 0 and len(s301_spell) == 0:
+    log(f"  ✅ AraSpell skip confirmed: spelling runs at 300, skipped at 301")
+    boundary_explanation = "Character count: <=300 runs AraSpell, >300 skips it"
+elif len(s300_spell) == len(s301_spell):
+    log(f"  ⚠ Same spelling count at both — boundary may not work as expected")
+    boundary_explanation = "Boundary NOT working as expected — same results at 300 and 301"
+else:
+    log(f"  ⚠ Different spelling counts but not the expected pattern")
+    boundary_explanation = f"Partial: 300={len(s300_spell)} spell, 301={len(s301_spell)} spell"
+
+# Test 2: Repetitive text (مرحبا × 100)
+repetitive = "مرحبا " * 100
+log(f"\n  Repetitive text: '{repetitive[:30]}...' ({len(repetitive)} chars)")
+r_rep = api_call("/api/analyze", repetitive)
+rep_corrected = r_rep.get("corrected", "")
+rep_sugg = r_rep.get("suggestions", [])
+rep_status = r_rep.get("status", "")
+
+# Check for garbling
+has_garble = any(c in rep_corrected for c in 'صطن') and 'مرحبا' not in rep_corrected[:20]
+log(f"  Status: {rep_status}, Suggestions: {len(rep_sugg)}")
+log(f"  Corrected starts with: '{rep_corrected[:60]}...'")
+if has_garble:
+    log(f"  ❌ GARBLED output detected")
+else:
+    log(f"  ✅ No obvious garbling")
+
+results['b4_boundary'] = {
+    'boundary_explanation': boundary_explanation,
+    'test_300': {'chars': 300, 'suggestions': len(s300), 'spelling': len(s300_spell)},
+    'test_301': {'chars': 301, 'suggestions': len(s301), 'spelling': len(s301_spell)},
+    'repetitive': {
+        'input_chars': len(repetitive),
+        'status': rep_status,
+        'suggestions': len(rep_sugg),
+        'garbled': has_garble,
+        'corrected_preview': rep_corrected[:100],
+    },
+}
+
+# ═══════════════════════════════════════════════════════════════
+# B5: Shadda Duplication Verification
+# ═══════════════════════════════════════════════════════════════
+log("\n" + "=" * 70)
+log("B5: Shadda Duplication — Sentence Context")
+log("=" * 70)
+
+shadda_tests = [
+    {"input": "إنّ العلم نور", "check": "إنّ", "desc": "إنّ in sentence"},
+    {"input": "علمت أنّ الامتحان صعب", "check": "أنّ", "desc": "أنّ in sentence"},
+    {"input": "إنّ", "check": "إنّ", "desc": "إنّ in isolation"},
+    {"input": "أنّ", "check": "أنّ", "desc": "أنّ in isolation"},
+]
+
+b5_results = []
+for t in shadda_tests:
+    r = api_call("/api/spelling", t["input"])
+    out = r.get("corrected_text", t["input"])
+    duplicated = out.count("إن") >= 2 or out.count("أن") >= 2
+    changed = out != t["input"]
+    status = "❌ DUPLICATED" if duplicated else ("⚠ CHANGED" if changed else "✅ OK")
+    b5_results.append({"input": t["input"], "output": out, "status": status})
+    log(f"  {t['desc']}: '{t['input']}' → '{out}' {status}")
+
+results['b5_shadda'] = b5_results
+
+# ═══════════════════════════════════════════════════════════════
+# B7: Unbalanced Brackets
+# ═══════════════════════════════════════════════════════════════
+log("\n" + "=" * 70)
+log("B7: Unbalanced Brackets — E6")
+log("=" * 70)
+
+bracket_tests = [
+    "(([{هذا النص}]))",
+    "({هذا النص})",
+    "(هذا النص)",
+    "[هذا النص]",
+]
+
+b7_results = []
+for text in bracket_tests:
+    r = api_call("/api/analyze", text)
+    corrected = r.get("corrected", text)
+    suggestions = r.get("suggestions", [])
+
+    # Count bracket balance
+    def bracket_balance(s):
+        opens = sum(1 for c in s if c in '([{')
+        closes = sum(1 for c in s if c in ')]}')
+        return opens, closes, opens == closes
+
+    in_o, in_c, in_bal = bracket_balance(text)
+    out_o, out_c, out_bal = bracket_balance(corrected)
+
+    result = {
+        "input": text, "corrected": corrected,
+        "input_balanced": in_bal, "output_balanced": out_bal,
+        "suggestions": len(suggestions),
+    }
+    b7_results.append(result)
+
+    if not out_bal and in_bal:
+        status = "❌ BRACKETS UNBALANCED"
+    elif out_bal:
+        status = "✅ BRACKETS OK"
+    elif not in_bal and not out_bal:
+        status = "⚠ BOTH UNBALANCED"
+    else:
+        status = "✅ FIXED"
+
+    log(f"  '{text}' → '{corrected}' {status}")
+    log(f"    Input: {in_o} opens, {in_c} closes, balanced={in_bal}")
+    log(f"    Output: {out_o} opens, {out_c} closes, balanced={out_bal}")
+
+results['b7_brackets'] = b7_results
+
+# ═══════════════════════════════════════════════════════════════
+# SAVE
+# ═══════════════════════════════════════════════════════════════
+output_path = os.path.join(os.path.dirname(__file__), 'round2_results.json')
+with open(output_path, 'w', encoding='utf-8') as f:
+    json.dump(results, f, ensure_ascii=False, indent=2)
+log(f"\nAll results saved to {output_path}")

tests/round2_results.json

@@ -0,0 +1,571 @@
+{
+  "timestamp": "2026-06-20T21:33:09.369155+00:00",
+  "a1_bug032": {
+    "input_chars": 982,
+    "input_words": 159,
+    "pipeline": {
+      "status": "success",
+      "suggestions": 4,
+      "elapsed_ms": 29556,
+      "timing_ms": {
+        "grammar_ms": 13132,
+        "punctuation_ms": 15452,
+        "spelling_ms": 0,
+        "total_ms": 28589
+      },
+      "warnings": null
+    },
+    "spelling": {
+      "elapsed_ms": 15980,
+      "error": null
+    },
+    "grammar": {
+      "elapsed_ms": 13068
+    },
+    "punctuation": {
+      "elapsed_ms": 21901
+    }
+  },
+  "a2_fp_rate": {
+    "original_appendix_e": {
+      "fp_count": 10,
+      "total": 10,
+      "fp_rate": "100%",
+      "results": [
+        {
+          "id": "R-01",
+          "input": "استوقفني المشهد فتأملته مليا",
+          "corrected": "استوقفني المشهد فتأملتة مليا",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-02",
+          "input": "تستأثر القوى العظمى بالنفوذ الدولي",
+          "corrected": "تستأثر القوى العظمى بالنفوذ الدولي.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-03",
+          "input": "استقطب المؤتمر ثلة من العلماء الأفذاذ",
+          "corrected": "استقطب المؤتمر ثلة من العلماء الأفذاذ.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-04",
+          "input": "يتسنى للمرء أن يستشف الحقيقة من بين السطور",
+          "corrected": "يتسنى للمرء أن يستشفِ الحقيقة من بين السطور",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-05",
+          "input": "ألقى المحاضر خطبة عصماء استحوذت على إعجاب الحاضرين",
+          "corrected": "ألقى المحاضر خطبة عصماء استحوذت؛ على إعجاب الحاضرين",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-06",
+          "input": "تمخض الاجتماع عن قرارات مصيرية",
+          "corrected": "تمخض الاجتماع عن قرارات مصيرية.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-07",
+          "input": "أرهقته المسغبة فاستكان للقدر",
+          "corrected": "أرهقته المسغبة فاستكان للقدر.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-08",
+          "input": "نستشرف آفاق المستقبل بثقة واقتدار",
+          "corrected": "نستشرف آفاق المستقبل بثقة واقتدار.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-09",
+          "input": "اعتراه القلق فتملكه الأرق",
+          "corrected": "اعتراه القلق فتملكة الأرق.",
+          "changed": true,
+          "suggestion_count": 2
+        },
+        {
+          "id": "R-10",
+          "input": "استأنف العمل بعد فترة من التقاعس",
+          "corrected": "استأنف العمل بعد فترة من التقاعس.",
+          "changed": true,
+          "suggestion_count": 1
+        }
+      ]
+    },
+    "constructed_sentences": {
+      "fp_count": 10,
+      "total": 10,
+      "fp_rate": "100%",
+      "results": [
+        {
+          "id": "R-01c",
+          "input": "المدينة العصماء تحتضن آلاف السكان",
+          "corrected": "المدينة العصماء تحتضن آلاف السكان.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-02c",
+          "input": "يستشف الباحث نتائج الدراسة بعناية",
+          "corrected": "يستشف الباحث نتائج الدراسة بعناية.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-03c",
+          "input": "أرهقته المسغبة والعطش الشديد",
+          "corrected": "أرهقته المسغبة والعطش الشديد!",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-04c",
+          "input": "التقاعس عن العمل يؤدي إلى الفشل",
+          "corrected": "التقاعس عن العمل يؤدي إلى الفشل.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-05c",
+          "input": "استئثار السلطة يهدد الديمقراطية",
+          "corrected": "استئثار السلطة يهدد الديمقراطية.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-06c",
+          "input": "تبجيل العلماء واجب على المجتمع",
+          "corrected": "تبجيل العلماء واجب على المجتمع.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-07c",
+          "input": "الرجل الدمث يحبه الجميع",
+          "corrected": "الرجل الدم يحبة الجميع.",
+          "changed": true,
+          "suggestion_count": 3
+        },
+        {
+          "id": "R-08c",
+          "input": "استقصاء الحقائق مهم في الصحافة",
+          "corrected": "استقصاء الحقائق مهم في الصحافة.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-09c",
+          "input": "لا يجوز التواني في طلب العلم",
+          "corrected": "لا يجوز التواني في طلب العلم.",
+          "changed": true,
+          "suggestion_count": 1
+        },
+        {
+          "id": "R-10c",
+          "input": "كتاب المستطرف من أمهات الكتب العربية",
+          "corrected": "كتاب المستطرف من أمهات الكتب العربية.",
+          "changed": true,
+          "suggestion_count": 1
+        }
+      ]
+    }
+  },
+  "b1_prefix": {
+    "total": 31,
+    "unchanged": 31,
+    "blocked": 0,
+    "leaked": 0,
+    "results": [
+      {
+        "word": "مدرسة",
+        "root": "مدرسة",
+        "prefix": "bare",
+        "input": "مدرسة مهم جدا",
+        "corrected": "مدرسة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "ومدرسة",
+        "root": "مدرسة",
+        "prefix": "wa",
+        "input": "ومدرسة مهم جدا",
+        "corrected": "ومدرسة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "بمدرسة",
+        "root": "مدرسة",
+        "prefix": "ba",
+        "input": "بمدرسة مهم جدا",
+        "corrected": "بمدرسة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "لمدرسة",
+        "root": "مدرسة",
+        "prefix": "la",
+        "input": "لمدرسة مهم جدا",
+        "corrected": "لمدرسة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "كمدرسة",
+        "root": "مدرسة",
+        "prefix": "ka",
+        "input": "كمدرسة مهم جدا",
+        "corrected": "كمدرسة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "شمس",
+        "root": "شمس",
+        "prefix": "bare",
+        "input": "شمس مهم جدا",
+        "corrected": "شمس مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "وشمس",
+        "root": "شمس",
+        "prefix": "wa",
+        "input": "وشمس مهم جدا",
+        "corrected": "وشمس مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "بشمس",
+        "root": "شمس",
+        "prefix": "ba",
+        "input": "بشمس مهم جدا",
+        "corrected": "بشمس مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "لشمس",
+        "root": "شمس",
+        "prefix": "la",
+        "input": "لشمس مهم جدا",
+        "corrected": "لشمس مهم جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "كشمس",
+        "root": "شمس",
+        "prefix": "ka",
+        "input": "كشمس مهم جدا",
+        "corrected": "كشمس مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "أمة",
+        "root": "أمة",
+        "prefix": "bare",
+        "input": "أمة مهم جدا",
+        "corrected": "أمة مهمة جدا؟",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "وأمة",
+        "root": "أمة",
+        "prefix": "wa",
+        "input": "وأمة مهم جدا",
+        "corrected": "وأمة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "بأمة",
+        "root": "أمة",
+        "prefix": "ba",
+        "input": "بأمة مهم جدا",
+        "corrected": "بأمة مهمة جدا؟",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "لأمة",
+        "root": "أمة",
+        "prefix": "la",
+        "input": "لأمة مهم جدا",
+        "corrected": "لأمة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "كأمة",
+        "root": "أمة",
+        "prefix": "ka",
+        "input": "كأمة مهم جدا",
+        "corrected": "كأمة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "نافذة",
+        "root": "نافذة",
+        "prefix": "bare",
+        "input": "نافذة مهم جدا",
+        "corrected": "نافذة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "ونافذة",
+        "root": "نافذة",
+        "prefix": "wa",
+        "input": "ونافذة مهم جدا",
+        "corrected": "ونافذة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "بنافذة",
+        "root": "نافذة",
+        "prefix": "ba",
+        "input": "بنافذة مهم جدا",
+        "corrected": "بنافذة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "لنافذة",
+        "root": "نافذة",
+        "prefix": "la",
+        "input": "لنافذة مهم جدا",
+        "corrected": "لنافذة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "كنافذة",
+        "root": "نافذة",
+        "prefix": "ka",
+        "input": "كنافذة مهم جدا",
+        "corrected": "كنافذة مهمة جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "علم",
+        "root": "علم",
+        "prefix": "bare",
+        "input": "علم مهم جدا",
+        "corrected": "علم مهم جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "وعلم",
+        "root": "علم",
+        "prefix": "wa",
+        "input": "وعلم مهم جدا",
+        "corrected": "وعلم مهم جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "بعلم",
+        "root": "علم",
+        "prefix": "ba",
+        "input": "بعلم مهم جدا",
+        "corrected": "بعلم مهم جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "لعلم",
+        "root": "علم",
+        "prefix": "la",
+        "input": "لعلم مهم جدا",
+        "corrected": "لعلم مهم جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "كعلم",
+        "root": "علم",
+        "prefix": "ka",
+        "input": "كعلم مهم جدا",
+        "corrected": "كعلم مهم جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "اقتصاد",
+        "root": "اقتصاد",
+        "prefix": "bare",
+        "input": "اقتصاد مهم جدا",
+        "corrected": "اقتصاد مهم جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "واقتصاد",
+        "root": "اقتصاد",
+        "prefix": "wa",
+        "input": "واقتصاد مهم جدا",
+        "corrected": "واقتصاد مهم جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "باقتصاد",
+        "root": "اقتصاد",
+        "prefix": "ba",
+        "input": "باقتصاد مهم جدا",
+        "corrected": "باقتصاد مهم جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "لاقتصاد",
+        "root": "اقتصاد",
+        "prefix": "la",
+        "input": "لاقتصاد مهم جدا",
+        "corrected": "لاقتصاد مهم جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "كاقتصاد",
+        "root": "اقتصاد",
+        "prefix": "ka",
+        "input": "كاقتصاد مهم جدا",
+        "corrected": "كاقتصاد مهم جدا.",
+        "word_preserved": true,
+        "targeting_suggestions": 0,
+        "status": "✅ PRESERVED"
+      },
+      {
+        "word": "ولذالك",
+        "input": "ولذالك يجب الاهتمام",
+        "corrected": "ولذالك يجب الاهتمام.",
+        "status": "⚠ UNCHANGED (misspelling preserved)"
+      }
+    ]
+  },
+  "b4_boundary": {
+    "boundary_explanation": "Partial: 300=23 spell, 301=2 spell",
+    "test_300": {
+      "chars": 300,
+      "suggestions": 31,
+      "spelling": 23
+    },
+    "test_301": {
+      "chars": 301,
+      "suggestions": 12,
+      "spelling": 2
+    },
+    "repetitive": {
+      "input_chars": 600,
+      "status": "success",
+      "suggestions": 4,
+      "garbled": false,
+      "corrected_preview": "مرحبا مرحبا مرحبا ومرحبا مرحبا مرحبامرحبا مرحبا مرحبا مرحب مرحبا مرحبا وسهلا مرحبا مرحبا ترحيبا مرحب"
+    }
+  },
+  "b5_shadda": [
+    {
+      "input": "إنّ العلم نور",
+      "output": "إن العلم نور",
+      "status": "⚠ CHANGED"
+    },
+    {
+      "input": "علمت أنّ الامتحان صعب",
+      "output": "علمت أن الامتحان صعب",
+      "status": "⚠ CHANGED"
+    },
+    {
+      "input": "إنّ",
+      "output": "إن إن",
+      "status": "❌ DUPLICATED"
+    },
+    {
+      "input": "أنّ",
+      "output": "أن أن",
+      "status": "❌ DUPLICATED"
+    }
+  ],
+  "b7_brackets": [
+    {
+      "input": "(([{هذا النص}]))",
+      "corrected": "( ( [ { هذا النص } ] ، و",
+      "input_balanced": true,
+      "output_balanced": false,
+      "suggestions": 1
+    },
+    {
+      "input": "({هذا النص})",
+      "corrected": "يُعدَّ هذا النصا",
+      "input_balanced": true,
+      "output_balanced": true,
+      "suggestions": 1
+    },
+    {
+      "input": "(هذا النص)",
+      "corrected": "هذا النص",
+      "input_balanced": true,
+      "output_balanced": true,
+      "suggestions": 1
+    },
+    {
+      "input": "[هذا النص]",
+      "corrected": "هذا النص",
+      "input_balanced": true,
+      "output_balanced": true,
+      "suggestions": 1
+    }
+  ]
+}

tests/test_bug_fixes.py

@@ -657,5 +657,186 @@ class TestSuffixCorruption(unittest.TestCase):
                          "Verb+pronoun كتبته→كتبتة must be blocked")
 
 
+# ═══════════════════════════════════════════════════════════════
+# P2: Ta Marbuta Fix Tests
+# ═══════════════════════════════════════════════════════════════
+class TestTaMarbutaFix(unittest.TestCase):
+    """Tests for the _fix_ta_marbuta pipeline function."""
+
+    @classmethod
+    def setUpClass(cls):
+        from app import _fix_ta_marbuta, _TA_MARBUTA_DICT
+        cls.fix = staticmethod(_fix_ta_marbuta)
+        cls.dict = _TA_MARBUTA_DICT
+
+    def test_basic_fix(self):
+        """المدرسه should be corrected to المدرسة."""
+        result, changes = self.fix('ذهبت الى المدرسه')
+        self.assertIn('المدرسة', result)
+        self.assertEqual(len(changes), 1)
+        self.assertEqual(changes[0]['original'], 'المدرسه')
+        self.assertEqual(changes[0]['correction'], 'المدرسة')
+
+    def test_multiple_fixes(self):
+        """Multiple ta marbuta errors in one sentence."""
+        result, changes = self.fix('الحياه في المدينه جميله')
+        self.assertIn('الحياة', result)
+        self.assertIn('المدينة', result)
+        self.assertGreaterEqual(len(changes), 2)
+
+    def test_no_false_positives(self):
+        """Words ending in ه that are NOT ta marbuta should be left alone."""
+        result, changes = self.fix('الله أكبر')
+        self.assertEqual(result, 'الله أكبر')
+        self.assertEqual(len(changes), 0)
+
+    def test_correct_text_untouched(self):
+        """Already correct text should not be changed."""
+        result, changes = self.fix('ذهبت إلى المدرسة')
+        self.assertEqual(result, 'ذهبت إلى المدرسة')
+        self.assertEqual(len(changes), 0)
+
+    def test_without_alef_lam(self):
+        """Bare words without ال should also be fixed."""
+        result, changes = self.fix('هذه مدرسه كبيره')
+        self.assertIn('مدرسة', result)
+
+    def test_dict_coverage(self):
+        """Dictionary should have significant coverage."""
+        self.assertGreater(len(self.dict), 50)
+
+
+# ═══════════════════════════════════════════════════════════════
+# P2: Hamza Whitelist Tests
+# ═══════════════════════════════════════════════════════════════
+class TestHamzaWhitelist(unittest.TestCase):
+    """Tests for hamza fix function."""
+
+    @classmethod
+    def setUpClass(cls):
+        try:
+            from nlp.spelling.araspell_rules import AraSpellPostProcessor
+            cls.fix = staticmethod(AraSpellPostProcessor.fix_common_hamza)
+            cls.available = True
+        except Exception:
+            cls.available = False
+
+    def test_anta_fix(self):
+        """انت should become أنت."""
+        if not self.available:
+            self.skipTest("AraSpellPostProcessor not available")
+        result = self.fix('انت طالب')
+        self.assertIn('أنت', result)
+
+    def test_ana_fix(self):
+        """انا should become أنا."""
+        if not self.available:
+            self.skipTest("AraSpellPostProcessor not available")
+        result = self.fix('انا ذاهب')
+        self.assertIn('أنا', result)
+
+    def test_alaan_fix(self):
+        """الان should become الآن."""
+        if not self.available:
+            self.skipTest("AraSpellPostProcessor not available")
+        result = self.fix('اذهب الان')
+        self.assertIn('الآن', result)
+
+    def test_correct_hamza_untouched(self):
+        """Already correct hamza should not be changed."""
+        if not self.available:
+            self.skipTest("AraSpellPostProcessor not available")
+        result = self.fix('أنت ذاهب إلى المدرسة')
+        self.assertEqual(result, 'أنت ذاهب إلى المدرسة')
+
+
+# ═══════════════════════════════════════════════════════════════
+# P3: Caching & Rate Limiting Tests
+# ═══════════════════════════════════════════════════════════════
+class TestCachingAndRateLimiting(unittest.TestCase):
+    """Tests for response caching and rate limiting."""
+
+    @classmethod
+    def setUpClass(cls):
+        from app import (
+            _get_cache_key, _get_cached_response,
+            _set_cached_response, _check_rate_limit,
+            _analyze_cache, _rate_limit_store
+        )
+        cls._get_cache_key = staticmethod(_get_cache_key)
+        cls._get_cached = staticmethod(_get_cached_response)
+        cls._set_cached = staticmethod(_set_cached_response)
+        cls._check_rate = staticmethod(_check_rate_limit)
+        cls._cache = _analyze_cache
+        cls._rate_store = _rate_limit_store
+
+    def setUp(self):
+        self._cache.clear()
+        self._rate_store.clear()
+
+    def test_cache_key_deterministic(self):
+        """Same text should produce same cache key."""
+        key1 = self._get_cache_key('مرحبا')
+        key2 = self._get_cache_key('مرحبا')
+        self.assertEqual(key1, key2)
+
+    def test_cache_key_different(self):
+        """Different texts should produce different keys."""
+        key1 = self._get_cache_key('مرحبا')
+        key2 = self._get_cache_key('أهلا')
+        self.assertNotEqual(key1, key2)
+
+    def test_cache_store_and_retrieve(self):
+        """Cached response should be retrievable."""
+        data = {'original': 'test', 'corrected': 'test', 'suggestions': []}
+        self._set_cached('مرحبا', data)
+        result = self._get_cached('مرحبا')
+        self.assertIsNotNone(result)
+        self.assertEqual(result['original'], 'test')
+
+    def test_cache_miss(self):
+        """Non-cached text should return None."""
+        result = self._get_cached('نص جديد')
+        self.assertIsNone(result)
+
+    def test_rate_limit_allows(self):
+        """First request should be allowed."""
+        self.assertTrue(self._check_rate('127.0.0.1'))
+
+    def test_rate_limit_blocks(self):
+        """Should block after exceeding limit."""
+        for _ in range(30):
+            self._check_rate('test_ip')
+        self.assertFalse(self._check_rate('test_ip'))
+
+
+# ═══════════════════════════════════════════════════════════════
+# P2: Grammar Splitting Tests
+# ═══════════════════════════════════════════════════════════════
+class TestGrammarSplitting(unittest.TestCase):
+    """Tests for grammar multi-word diff splitting logic."""
+
+    def test_split_logic(self):
+        """Multi-word grammar diffs should be split into individual words."""
+        # Simulate the splitting logic from analyze_text
+        orig_text = 'الي المدرسه الاستاذ'
+        corr_text = 'إلى المدرسة الأستاذ'
+        orig_words = orig_text.split()
+        corr_words = corr_text.split()
+
+        self.assertEqual(len(orig_words), len(corr_words))
+
+        diffs = []
+        for ow, cw in zip(orig_words, corr_words):
+            if ow != cw:
+                diffs.append({'original': ow, 'correction': cw})
+
+        self.assertEqual(len(diffs), 3)
+        self.assertEqual(diffs[0]['original'], 'الي')
+        self.assertEqual(diffs[0]['correction'], 'إلى')
+        self.assertEqual(diffs[1]['original'], 'المدرسه')
+        self.assertEqual(diffs[1]['correction'], 'المدرسة')
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_concurrency.py

@@ -0,0 +1,176 @@
+"""
+Phase 9 — Concurrency Re-verification
+
+Send 5 genuinely different inputs simultaneously.
+Verify each response correctly corresponds to its own input.
+No mixed, swapped, or cross-contaminated suggestions.
+
+If cross-contamination is found, treat as P0 bug.
+"""
+import sys, os, json, time, requests
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+API_BASE = "https://bayan10-bayan-api.hf.space"
+TIMEOUT = 60
+
+def api_call(endpoint, text):
+    url = f"{API_BASE}{endpoint}"
+    try:
+        t0 = time.time()
+        resp = requests.post(url, json={"text": text}, timeout=TIMEOUT)
+        elapsed = int((time.time() - t0) * 1000)
+        if resp.status_code == 200:
+            data = resp.json()
+            data['_elapsed_ms'] = elapsed
+            return data
+        return {"error": f"HTTP {resp.status_code}", "_elapsed_ms": elapsed}
+    except Exception as e:
+        return {"error": f"{type(e).__name__}: {e}"}
+
+
+# 5 genuinely different inputs — different lengths, different error types
+CONCURRENT_INPUTS = [
+    {
+        "id": "CONC-1",
+        "text": "الحديقه جميله",
+        "description": "Short text with spelling error (ه→ة)",
+        "expected_contains": "الحديق",  # at least part of the input
+        "must_not_contain_from_others": ["المدرسة", "القاهرة", "مصر"],
+    },
+    {
+        "id": "CONC-2",
+        "text": "الطلاب ذهبو الى المدرسة",
+        "description": "Medium text with grammar error (ذهبو→ذهبوا)",
+        "expected_contains": "المدرسة",
+        "must_not_contain_from_others": ["الحديق", "القاهرة عاصمة"],
+    },
+    {
+        "id": "CONC-3",
+        "text": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة في بناء كتلة عضلية قوية ويا له من التزام حديدي يثير الإعجاب ولقد كان أداؤه في المسابقات الأخيرة مبهرا للغاية",
+        "description": "Long text with punctuation needed (50+ words)",
+        "expected_contains": "الرياضي",
+        "must_not_contain_from_others": ["الحديق", "المدرسة"],
+    },
+    {
+        "id": "CONC-4",
+        "text": "القاهرة عاصمة مصر",
+        "description": "Correct text (should return ~0 suggestions)",
+        "expected_contains": "القاهرة",
+        "must_not_contain_from_others": ["الحديق", "المدرسة", "الرياضي"],
+    },
+    {
+        "id": "CONC-5",
+        "text": "هذة المدينه جميله جدا ومناخها معتدل",
+        "description": "Text with mixed errors (هذة→هذه, ه→ة)",
+        "expected_contains": "المدين",
+        "must_not_contain_from_others": ["المدرسة", "القاهرة", "الرياضي"],
+    },
+]
+
+
+def test_concurrent():
+    print("=" * 70)
+    print("PHASE 9 — Concurrency Re-verification")
+    print("=" * 70)
+
+    # Fire all 5 requests simultaneously
+    results = {}
+    print(f"\nSending {len(CONCURRENT_INPUTS)} requests simultaneously...")
+
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        futures = {}
+        for inp in CONCURRENT_INPUTS:
+            future = executor.submit(api_call, "/api/analyze", inp["text"])
+            futures[future] = inp
+
+        for future in as_completed(futures):
+            inp = futures[future]
+            result = future.result()
+            results[inp["id"]] = {
+                "input": inp,
+                "response": result,
+            }
+            print(f"  {inp['id']} completed ({result.get('_elapsed_ms', '?')}ms)")
+
+    # Verify each response corresponds to its own input
+    print("\n--- Verification ---")
+    all_pass = True
+    contamination_found = False
+
+    for test_id, data in sorted(results.items()):
+        inp = data["input"]
+        resp = data["response"]
+
+        if "error" in resp and "status" not in resp:
+            print(f"\n  ⚠ {test_id}: ERROR — {resp['error']}")
+            continue
+
+        corrected = resp.get("corrected", "")
+        original = resp.get("original", "")
+        suggestions = resp.get("suggestions", [])
+
+        print(f"\n  {test_id}: {inp['description']}")
+        print(f"    Input:     '{inp['text'][:60]}...'")
+        print(f"    Original:  '{original[:60]}...'")
+        print(f"    Corrected: '{corrected[:60]}...'")
+        print(f"    Suggestions: {len(suggestions)}")
+
+        # Check 1: original field should match our input
+        if original != inp["text"]:
+            print(f"    ❌ FAIL: original != input! (cross-contamination?)")
+            contamination_found = True
+            all_pass = False
+        else:
+            print(f"    ✓ original matches input")
+
+        # Check 2: corrected should contain expected content
+        if inp["expected_contains"] in corrected:
+            print(f"    ✓ corrected contains '{inp['expected_contains']}'")
+        else:
+            print(f"    ⚠ corrected missing '{inp['expected_contains']}'")
+
+        # Check 3: corrected must NOT contain content from other inputs
+        for foreign in inp["must_not_contain_from_others"]:
+            if foreign in corrected:
+                print(f"    ❌ CONTAMINATION: corrected contains '{foreign}' from another input!")
+                contamination_found = True
+                all_pass = False
+
+        # Check 4: suggestions should reference text in our input
+        for s in suggestions:
+            s_orig = s.get("original", "")
+            s_start = s.get("start", 0)
+            s_end = s.get("end", 0)
+            # The suggestion's original text should be a substring of our input
+            if s_orig and s_orig not in inp["text"]:
+                # Check if it's a substring match (punc may include partial words)
+                input_slice = inp["text"][s_start:s_end]
+                if s_orig != input_slice:
+                    print(f"    ⚠ Suggestion '{s_orig}' [{s_start}:{s_end}] not in input")
+
+    print("\n" + "=" * 50)
+    if contamination_found:
+        print("🚨 P0: CROSS-CONTAMINATION DETECTED!")
+        print("   PipelineContext state is leaking between requests.")
+        print("   STOP ALL OTHER WORK AND FIX THIS FIRST.")
+    elif all_pass:
+        print("✅ ALL PASSED — No cross-contamination detected.")
+    else:
+        print("⚠ Some checks failed but no cross-contamination.")
+
+    return {
+        "test_count": len(CONCURRENT_INPUTS),
+        "all_pass": all_pass,
+        "contamination_found": contamination_found,
+        "results": {k: {"corrected": v["response"].get("corrected", ""),
+                        "suggestions_count": len(v["response"].get("suggestions", []))}
+                   for k, v in results.items()},
+    }
+
+
+if __name__ == "__main__":
+    result = test_concurrent()
+    output_path = os.path.join(os.path.dirname(__file__), 'phase9_results.json')
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(result, f, ensure_ascii=False, indent=2)
+    print(f"\nResults saved to {output_path}")

trace_punc.py

@@ -0,0 +1,73 @@
+import sys, os, re
+sys.path.insert(0, 'src')
+import logging; logging.basicConfig(level=logging.INFO)
+print("Starting...")
+
+import torch
+print(f"CUDA available: {torch.cuda.is_available()}")
+
+from transformers import EncoderDecoderModel, AutoTokenizer
+print("Loading PuncAra-v1...")
+model = EncoderDecoderModel.from_pretrained("bayan10/PuncAra-v1")
+tokenizer = AutoTokenizer.from_pretrained("bayan10/PuncAra-v1")
+model.eval()
+print("Model loaded!")
+
+inp = "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة في بناء كتلة عضلية قوية ويا له من التزام حديدي يثير الإعجاب"
+print(f"\nINPUT: {inp}")
+
+# Raw inference
+from nlp.punctuation.punctuation_rules import arabic_preprocessing
+processed = arabic_preprocessing(inp)
+inputs = tokenizer(processed, return_tensors="pt", padding=True, truncation=True, max_length=128)
+print("Running inference...")
+with torch.no_grad():
+    outputs = model.generate(
+        inputs.input_ids,
+        attention_mask=inputs.attention_mask,
+        decoder_start_token_id=tokenizer.cls_token_id,
+        bos_token_id=tokenizer.cls_token_id,
+        eos_token_id=tokenizer.sep_token_id,
+        pad_token_id=tokenizer.pad_token_id,
+        max_length=128, num_beams=3, repetition_penalty=1.2,
+        length_penalty=1.0, early_stopping=True, do_sample=False
+    )
+raw = tokenizer.decode(outputs[0], skip_special_tokens=True)
+print(f"[A] RAW MODEL: {raw}")
+
+# Strip non-punc
+from nlp.punctuation.punctuation_service import PunctuationChecker
+checker = PunctuationChecker(model, tokenizer, torch.device('cpu'))
+stripped = checker._strip_non_punctuation_changes(inp, raw)
+print(f"[B] STRIPPED:   {stripped}")
+if stripped != raw:
+    rw, sw = raw.split(), stripped.split()
+    for w1, w2 in zip(rw, sw):
+        if w1 != w2:
+            print(f"    LOST: '{w1}' -> '{w2}'")
+
+# Postprocess
+from nlp.punctuation.punctuation_rules import arabic_postprocessing
+final = arabic_postprocessing(stripped)
+print(f"[C] FINAL:      {final}")
+
+# Diffs
+from app import get_word_diffs
+from nlp.punctuation.punctuation_rules import validate_punctuation_diff
+if final != inp:
+    diffs = get_word_diffs(inp, final)
+    print(f"[D] DIFFS ({len(diffs)}):")
+    for d in diffs:
+        o, c = d.get('original',''), d.get('correction','')
+        valid = validate_punctuation_diff(d)
+        oa = re.sub(r'[^\u0600-\u06FFa-zA-Z]','',o)
+        ca = re.sub(r'[^\u0600-\u06FFa-zA-Z]','',c)
+        alpha_ok = oa == ca
+        s = "PASS" if valid and alpha_ok else "BLOCKED"
+        r = ""
+        if not valid: r += " safety"
+        if not alpha_ok: r += " alpha"
+        print(f"    [{d['start']}:{d['end']}] '{o}' -> '{c}'  [{s}{r}]")
+else:
+    print("[D] NO DIFFS!")
+print("\nDONE")

trace_punctuation.py

@@ -0,0 +1,176 @@
+"""
+BAYAN Punctuation Trace — Diagnose where punctuation marks get lost.
+
+Compares:
+  A) Raw PuncAra model output (no pipeline)
+  B) After _strip_non_punctuation_changes (Fix P1)
+  C) After get_word_diffs (diff algorithm)
+  D) After StageLocker check
+  E) After validate_punctuation_diff (safety layer)
+  F) After overlap resolver + patch cap
+"""
+
+import sys, os, re, difflib
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
+
+# Suppress model loading noise
+import logging
+logging.basicConfig(level=logging.WARNING)
+
+# ─── Test Sentences ─────────────────────────────────────────────
+TEST_SENTENCES = [
+    {
+        "input": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة في بناء كتلة عضلية قوية ويا له من التزام حديدي يثير الإعجاب",
+        "expected": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة؛ رغبة في بناء كتلة عضلية قوية، ويا له من التزام حديدي يثير الإعجاب!",
+    },
+    {
+        "input": "كانت الفتيات يلعبن في الحديقة وفجأة سقطت إحداهن وبدأت تبكي بشدة",
+        "expected": "كانت الفتيات يلعبن في الحديقة، وفجأة سقطت إحداهن وبدأت تبكي بشدة.",
+    },
+    {
+        "input": "إن الذكاء الاصطناعي يلعب دورا هاما لذلك يجب الاهتمام به",
+        "expected": "إن الذكاء الاصطناعي يلعب دورا هاما؛ لذلك يجب الاهتمام به.",
+    },
+    {
+        "input": "هل تعلم أن القاهرة هي عاصمة مصر وتقع على ضفاف نهر النيل",
+        "expected": "هل تعلم أن القاهرة هي عاصمة مصر، وتقع على ضفاف نهر النيل؟",
+    },
+    {
+        "input": "قال المعلم للطلاب ادرسوا جيدا فالامتحان قريب",
+        "expected": "قال المعلم للطلاب: ادرسوا جيدا، فالامتحان قريب.",
+    },
+]
+
+def count_punct(text):
+    """Count punctuation marks in text."""
+    marks = set('.,;:!?،؛؟')
+    return sum(1 for c in text if c in marks)
+
+def diff_punct(before, after):
+    """Show what punctuation marks were added/removed."""
+    marks = set('.,;:!?،؛؟')
+    before_marks = [(i, c) for i, c in enumerate(before) if c in marks]
+    after_marks = [(i, c) for i, c in enumerate(after) if c in marks]
+    return before_marks, after_marks
+
+def main():
+    print("=" * 80)
+    print("BAYAN PUNCTUATION TRACE — Where do punctuation marks get lost?")
+    print("=" * 80)
+
+    # Load model
+    print("\n[1/2] Loading PuncAra-v1 model...")
+    from nlp.punctuation.punctuation_service import get_punctuation_model, PunctuationChecker
+    punc_checker = get_punctuation_model()
+    print("  ✓ Model loaded\n")
+
+    # Load pipeline tools
+    print("[2/2] Loading pipeline tools...")
+    from app import get_word_diffs
+    from nlp.punctuation.punctuation_rules import validate_punctuation_diff
+    print("  ✓ Tools loaded\n")
+
+    for idx, test in enumerate(TEST_SENTENCES):
+        inp = test["input"]
+        expected = test["expected"]
+        
+        print("─" * 80)
+        print(f"TEST {idx+1}")
+        print(f"  INPUT:    {inp}")
+        print(f"  EXPECTED: {expected}")
+        print(f"  Expected marks: {count_punct(expected)}")
+        print()
+
+        # ─── Stage A: Raw model output (no post-processing) ────────
+        raw_output = punc_checker._fix_punctuation(inp)
+        print(f"  [A] RAW MODEL:     {raw_output}")
+        print(f"      Marks added:   {count_punct(raw_output) - count_punct(inp)}")
+        print()
+
+        # ─── Stage B: After _strip_non_punctuation_changes ─────────
+        stripped = punc_checker._strip_non_punctuation_changes(inp, raw_output)
+        print(f"  [B] STRIP NON-PUNC: {stripped}")
+        if stripped != raw_output:
+            print(f"      ⚠ Changes stripped! Diff from raw:")
+            for w1, w2 in zip(raw_output.split(), stripped.split()):
+                if w1 != w2:
+                    print(f"        '{w1}' → '{w2}'")
+        print(f"      Marks added:   {count_punct(stripped) - count_punct(inp)}")
+        print()
+
+        # ─── Stage C: get_word_diffs ───────────────────────────────
+        # This is what correct() returns after postprocessing
+        from nlp.punctuation.punctuation_rules import arabic_postprocessing
+        final_punc = arabic_postprocessing(stripped)
+        
+        print(f"  [C] FINAL PUNC OUT: {final_punc}")
+        print(f"      Marks added:   {count_punct(final_punc) - count_punct(inp)}")
+        print()
+
+        # ─── Stage D: Word diffs ──────────────────────────────────
+        if final_punc != inp:
+            diffs = get_word_diffs(inp, final_punc)
+            print(f"  [D] WORD DIFFS ({len(diffs)} found):")
+            for d in diffs:
+                orig = d.get('original', '')
+                corr = d.get('correction', '')
+                
+                # Check validate_punctuation_diff
+                is_valid = validate_punctuation_diff(d)
+                
+                # Check alpha match (lock bypass)
+                orig_alpha = re.sub(r'[^\u0600-\u06FFa-zA-Z]', '', orig)
+                corr_alpha = re.sub(r'[^\u0600-\u06FFa-zA-Z]', '', corr)
+                alpha_match = orig_alpha == corr_alpha
+                
+                status_parts = []
+                if not is_valid:
+                    status_parts.append("❌ SAFETY-REJECTED")
+                if not alpha_match:
+                    status_parts.append("❌ LOCK-BLOCKED (alpha differs)")
+                if is_valid and alpha_match:
+                    status_parts.append("✅ WOULD PASS")
+                elif is_valid:
+                    status_parts.append("✅ valid-punc")
+                
+                status = " | ".join(status_parts)
+                print(f"      [{d['start']}:{d['end']}] '{orig}' → '{corr}'  {status}")
+        else:
+            print(f"  [D] NO DIFFS — model returned same text as input!")
+        
+        print()
+
+    # ─── Summary ───────────────────────────────────────────────────
+    print("=" * 80)
+    print("LOSS POINTS SUMMARY")
+    print("=" * 80)
+    print("""
+Where punctuation marks can be lost:
+
+  [A→B] _strip_non_punctuation_changes():
+         If model changes a word's spelling AND adds punctuation,
+         the punctuation transfer logic may fail.
+
+  [B→C] arabic_postprocessing():
+         Typographic cleanup may remove valid marks.
+
+  [C→D] get_word_diffs():
+         Word-level diff may merge/split changes incorrectly.
+
+  [D→E] StageLocker:
+         Locked ranges from spelling/grammar block nearby punctuation.
+         (Now relaxed: pure-punc changes pass through)
+
+  [D→E] validate_punctuation_diff():
+         Safety layer rejects diffs that change Arabic text.
+
+  [E→F] Overlap resolver:
+         Grammar/spelling patches take priority over punctuation.
+
+  [E→F] Patch cap:
+         Max 3 punctuation patches per response.
+""")
+
+
+if __name__ == "__main__":
+    main()