revert: restore backend to 1a76471, keep all UI/UX improvements

.github/workflows/deploy.yml

@@ -7,30 +7,8 @@ on:
     branches: [main]
 
 jobs:
-  test:
-    name: Run Regression Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-          cache: 'pip'
-
-      - name: Install dependencies
-        run: |
-          pip install flask flask-cors python-dotenv pytest
-          pip install -r requirements.txt || true
-
-      - name: Run pipeline tests
-        run: |
-          cd src && python -m pytest ../tests/test_pipeline_hardening.py ../tests/test_bug_fixes.py -v --tb=short
-
   lint-and-validate:
     name: Validate Code
-    needs: test
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

hybrid_module.py

@@ -1,147 +0,0 @@
-
-# hybrid_module.py
-
-import torch
-import pickle
-from transformers import GPT2LMHeadModel, GPT2Tokenizer
-from huggingface_hub import hf_hub_download
-
-# ---------- Load Bigram ----------
-
-def load_bigram(repo_id="bayan10/AutoComplete", filename="bigram_model_v4.pkl"):
-    path = hf_hub_download(repo_id=repo_id, filename=filename)
-    with open(path, "rb") as f:
-        data = pickle.load(f)
-    return data["unigrams"], data["bigrams"]
-
-# ---------- Load GPT-2 ----------
-def load_gpt2(model_name="aubmindlab/aragpt2-base"):
-    tokenizer = GPT2Tokenizer.from_pretrained(model_name)
-    model = GPT2LMHeadModel.from_pretrained(model_name)
-    tokenizer.pad_token = tokenizer.eos_token
-    model.config.pad_token_id = tokenizer.eos_token_id
-    model.eval()
-    return tokenizer, model
-
-# ---------- GPT-2 scoring ----------
-def gpt2_next_token_probs(prefix, tokenizer, model, top_k=50):
-    inputs = tokenizer(
-        prefix,
-        return_tensors="pt",
-        truncation=True,
-        max_length=1024
-    )
-
-    with torch.no_grad():
-        outputs = model(**inputs)
-        logits = outputs.logits[0, -1]
-
-    probs = torch.softmax(logits, dim=-1)
-    top_probs, top_ids = torch.topk(probs, top_k)
-
-    prob_dict = {}
-    for idx, prob in zip(top_ids, top_probs):
-        word = tokenizer.decode([idx]).strip()
-        if word:
-            prob_dict[word] = prob.item()
-
-    return prob_dict
-
-# ---------- Statistical autocomplete ----------
-
-
-def statistical_autocomplete(text, unigrams, bigrams, top_k=20):
-    tokens = text.strip().split()
-    if not tokens:
-        return []
-
-    last_word = tokens[-1]
-    candidates = []
-
-    if last_word in bigrams:
-        for w, c in bigrams[last_word].items():
-            if len(w) < 3 or w == last_word:
-                continue
-            candidates.append((w, c))
-
-    if not candidates:
-        for w, c in unigrams.items():
-            if len(w) < 3:
-                continue
-            candidates.append((w, c))
-
-    total = sum(c for _, c in candidates)
-    preds = [(w, c / total) for w, c in candidates]
-    preds.sort(key=lambda x: x[1], reverse=True)
-    preds = merge_similar_predictions(preds, top_k=top_k)
-    return preds[:top_k]
-
-# ---------- Hybrid autocomplete ----------
-def hybrid_autocomplete(prefix, unigrams, bigrams, tokenizer, model, alpha=0.6, k=5):
-    words = prefix.strip().split()
-    if len(words) < 1:
-        return []
-
-    last_word = words[-1]
-    if last_word not in bigrams:
-        return []
-
-    # -------- Statistical (Bigram) --------
-    stat_candidates = statistical_autocomplete(
-    prefix,
-    unigrams,
-    bigrams,
-    top_k=20
-)
-
-    # -------- Neural (GPT-2) — ONCE --------
-    gpt2_probs = gpt2_next_token_probs(prefix, tokenizer, model, top_k=50)
-
-    # -------- Hybrid scoring --------
-    results = []
-    for w, stat_p in stat_candidates:
-        neural_p = gpt2_probs.get(w, 1e-8)  # small value if not found
-        score = alpha * stat_p + (1 - alpha) * neural_p
-        results.append((w, score))
-
-    return sorted(results, key=lambda x: x[1], reverse=True)[:k]
-
-import re
-from collections import defaultdict
-
-def canonical_form(word):
-    word = re.sub("[إأآا]", "ا", word)
-    word = re.sub("ى", "ي", word)
-    word = re.sub("ؤ", "و", word)
-    word = re.sub("ئ", "ي", word)
-    word = re.sub("ة", "ه", word)
-    word = re.sub(r"[ًٌٍَُِّْ]", "", word)
-    return word
-
-
-
-def merge_similar_predictions(preds, top_k=20):
-    groups = defaultdict(lambda: {"score": 0.0, "words": []})
-
-    for w, p in preds:
-        key = canonical_form(w)
-        groups[key]["score"] += p
-        groups[key]["words"].append(w)
-
-    merged = sorted(
-        groups.values(),
-        key=lambda x: x["score"],
-        reverse=True
-    )
-
-    return [
-        (group["words"][0], group["score"])
-        for group in merged[:top_k]
-    ]
-
-
-
-
-
-
-

src/app.py

@@ -831,14 +831,6 @@ def _is_small_spelling_change(orig_word, corr_word, vocab_manager=None):
         if corr_word == orig_word[:-1] or len(corr_word) < len(orig_word):
             return 0.0
 
-    # ── GUARD 3: Word truncation protection (Phase 4, P3) ──
-    # Reject corrections that significantly shorten a valid word.
-    # E.g. الدمث→الدم (rare word truncated to different word).
-    if vocab_manager and len(orig_word) - len(corr_word) >= 2:
-        if vocab_manager.is_iv(orig_word):
-            logger.info(f"[SPELLING] Blocked truncation: '{orig_word}'→'{corr_word}' (valid word shortened by {len(orig_word) - len(corr_word)} chars)")
-            return 0.0
-
     # CRITICAL: If both words are valid Arabic words, only accept known fixes.
     # This prevents the spelling model from changing one correct word to another
     # (e.g. وكان→وكأن, which changes "and was" to "as if" — a meaning change).
@@ -1338,26 +1330,6 @@ def analyze_text():
                 logger.error(traceback.format_exc())
                 timing_ms['spelling_error'] = f"{type(e).__name__}: {str(e)[:200]}"
 
-        # ── Standalone hamza-fix pass (P1/P2) ──
-        # fix_common_hamza handles common hamza errors that AraSpell missed
-        # in sentence context: انت→أنت, الان→الآن, ام→أم, انا→أنا, etc.
-        # This runs on the FULL text regardless of whether AraSpell ran.
-        try:
-            from nlp.spelling.araspell_rules import AraSpellPostProcessor
-            hamza_fixed = AraSpellPostProcessor.fix_common_hamza(current_text)
-            if hamza_fixed != current_text:
-                hamza_diffs = get_word_diffs(current_text, hamza_fixed)
-                for hd in hamza_diffs:
-                    ctx.add_patch(
-                        'spelling', hd['start'], hd['end'],
-                        hd['correction'], confidence=0.95,
-                    )
-                    logger.info(f"[HAMZA-FIX] '{hd.get('original','')}' → '{hd.get('correction','')}'")
-                ctx.mutate_text(hamza_fixed, OffsetMapper)
-                current_text = ctx.current_text
-        except Exception as e:
-            logger.error(f"[ANALYZE] Hamza fix failed: {type(e).__name__}: {e}")
-
         # 2. Grammar (runs on spelling-corrected text — word-level dependency)
         try:
             t0 = time.time()
@@ -1369,40 +1341,7 @@ def analyze_text():
             logger.info(f"[ANALYZE] Step 2: Grammar done in {timing_ms['grammar_ms']}ms")
             if corrected_grammar != ctx.current_text:
                 diffs = get_word_diffs(ctx.current_text, corrected_grammar)
-
-                # ── Phase 2 (P1): Split multi-word grammar diffs ──
-                # Break merged diffs like "الي المدرسه الاستاذ"→"إلى المدرسة الأستاذ"
-                # into individual word-level suggestions.
-                expanded_diffs = []
                 for d in diffs:
-                    orig_text = d.get('original', '')
-                    corr_text = d.get('correction', '')
-                    orig_words_list = orig_text.split()
-                    corr_words_list = corr_text.split()
-                    # Only split if same word count and multiple words
-                    if (len(orig_words_list) > 1
-                            and len(orig_words_list) == len(corr_words_list)):
-                        # Split into per-word diffs
-                        search_from = d['start']
-                        for ow, cw in zip(orig_words_list, corr_words_list):
-                            if ow != cw:
-                                w_start = ctx.current_text.find(ow, search_from)
-                                if w_start >= 0:
-                                    w_end = w_start + len(ow)
-                                    expanded_diffs.append({
-                                        'start': w_start, 'end': w_end,
-                                        'original': ow, 'correction': cw,
-                                        'type': 'generic'
-                                    })
-                                    search_from = w_end
-                                else:
-                                    search_from += len(ow) + 1
-                            else:
-                                search_from += len(ow) + 1
-                    else:
-                        expanded_diffs.append(d)
-
-                for d in expanded_diffs:
                     # StageLocker: skip diffs that overlap with locked ranges
                     if ctx.stage_locker.is_locked(d['start'], d['end']):
                         logger.info(
@@ -1411,25 +1350,6 @@ def analyze_text():
                         )
                         continue
 
-                    # ── Punctuation-protection guard ──
-                    # Grammar model sometimes strips/normalizes punctuation
-                    # (removes full stops, commas, etc.). Reject grammar diffs
-                    # that only differ in punctuation marks — grammar should
-                    # only fix grammar, not touch punctuation.
-                    _orig_t = d.get('original', '')
-                    _corr_t = d.get('correction', '')
-                    _PUNC_CHARS = set('.,;:!?؟،؛。·…•–—\u200f\u200e')
-                    _orig_no_punc = ''.join(c for c in _orig_t if c not in _PUNC_CHARS)
-                    _corr_no_punc = ''.join(c for c in _corr_t if c not in _PUNC_CHARS)
-                    if _orig_no_punc == _corr_no_punc:
-                        # Only punctuation was changed — block it
-                        if _orig_t != _corr_t:
-                            logger.info(
-                                f"[GRAMMAR] Blocked punctuation change: "
-                                f"'{_orig_t}' → '{_corr_t}' (grammar must not alter punctuation)"
-                            )
-                            continue
-
                     # Reject grammar hallucinations (e.g. جالس→جاكسون)
                     orig_text = d.get('original', '')
                     corr_text = d.get('correction', '')

trace_punc.py

@@ -1,73 +0,0 @@
-import sys, os, re
-sys.path.insert(0, 'src')
-import logging; logging.basicConfig(level=logging.INFO)
-print("Starting...")
-
-import torch
-print(f"CUDA available: {torch.cuda.is_available()}")
-
-from transformers import EncoderDecoderModel, AutoTokenizer
-print("Loading PuncAra-v1...")
-model = EncoderDecoderModel.from_pretrained("bayan10/PuncAra-v1")
-tokenizer = AutoTokenizer.from_pretrained("bayan10/PuncAra-v1")
-model.eval()
-print("Model loaded!")
-
-inp = "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة في بناء كتلة عضلية قوية ويا له من التزام حديدي يثير الإعجاب"
-print(f"\nINPUT: {inp}")
-
-# Raw inference
-from nlp.punctuation.punctuation_rules import arabic_preprocessing
-processed = arabic_preprocessing(inp)
-inputs = tokenizer(processed, return_tensors="pt", padding=True, truncation=True, max_length=128)
-print("Running inference...")
-with torch.no_grad():
-    outputs = model.generate(
-        inputs.input_ids,
-        attention_mask=inputs.attention_mask,
-        decoder_start_token_id=tokenizer.cls_token_id,
-        bos_token_id=tokenizer.cls_token_id,
-        eos_token_id=tokenizer.sep_token_id,
-        pad_token_id=tokenizer.pad_token_id,
-        max_length=128, num_beams=3, repetition_penalty=1.2,
-        length_penalty=1.0, early_stopping=True, do_sample=False
-    )
-raw = tokenizer.decode(outputs[0], skip_special_tokens=True)
-print(f"[A] RAW MODEL: {raw}")
-
-# Strip non-punc
-from nlp.punctuation.punctuation_service import PunctuationChecker
-checker = PunctuationChecker(model, tokenizer, torch.device('cpu'))
-stripped = checker._strip_non_punctuation_changes(inp, raw)
-print(f"[B] STRIPPED:   {stripped}")
-if stripped != raw:
-    rw, sw = raw.split(), stripped.split()
-    for w1, w2 in zip(rw, sw):
-        if w1 != w2:
-            print(f"    LOST: '{w1}' -> '{w2}'")
-
-# Postprocess
-from nlp.punctuation.punctuation_rules import arabic_postprocessing
-final = arabic_postprocessing(stripped)
-print(f"[C] FINAL:      {final}")
-
-# Diffs
-from app import get_word_diffs
-from nlp.punctuation.punctuation_rules import validate_punctuation_diff
-if final != inp:
-    diffs = get_word_diffs(inp, final)
-    print(f"[D] DIFFS ({len(diffs)}):")
-    for d in diffs:
-        o, c = d.get('original',''), d.get('correction','')
-        valid = validate_punctuation_diff(d)
-        oa = re.sub(r'[^\u0600-\u06FFa-zA-Z]','',o)
-        ca = re.sub(r'[^\u0600-\u06FFa-zA-Z]','',c)
-        alpha_ok = oa == ca
-        s = "PASS" if valid and alpha_ok else "BLOCKED"
-        r = ""
-        if not valid: r += " safety"
-        if not alpha_ok: r += " alpha"
-        print(f"    [{d['start']}:{d['end']}] '{o}' -> '{c}'  [{s}{r}]")
-else:
-    print("[D] NO DIFFS!")
-print("\nDONE")

trace_punctuation.py

@@ -1,176 +0,0 @@
-"""
-BAYAN Punctuation Trace — Diagnose where punctuation marks get lost.
-
-Compares:
-  A) Raw PuncAra model output (no pipeline)
-  B) After _strip_non_punctuation_changes (Fix P1)
-  C) After get_word_diffs (diff algorithm)
-  D) After StageLocker check
-  E) After validate_punctuation_diff (safety layer)
-  F) After overlap resolver + patch cap
-"""
-
-import sys, os, re, difflib
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
-
-# Suppress model loading noise
-import logging
-logging.basicConfig(level=logging.WARNING)
-
-# ─── Test Sentences ─────────────────────────────────────────────
-TEST_SENTENCES = [
-    {
-        "input": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة رغبة في بناء كتلة عضلية قوية ويا له من التزام حديدي يثير الإعجاب",
-        "expected": "التزم الرياضي بتناول وجباته الصحية وحساب سعراته بدقة؛ رغبة في بناء كتلة عضلية قوية، ويا له من التزام حديدي يثير الإعجاب!",
-    },
-    {
-        "input": "كانت الفتيات يلعبن في الحديقة وفجأة سقطت إحداهن وبدأت تبكي بشدة",
-        "expected": "كانت الفتيات يلعبن في الحديقة، وفجأة سقطت إحداهن وبدأت تبكي بشدة.",
-    },
-    {
-        "input": "إن الذكاء الاصطناعي يلعب دورا هاما لذلك يجب الاهتمام به",
-        "expected": "إن الذكاء الاصطناعي يلعب دورا هاما؛ لذلك يجب الاهتمام به.",
-    },
-    {
-        "input": "هل تعلم أن القاهرة هي عاصمة مصر وتقع على ضفاف نهر النيل",
-        "expected": "هل تعلم أن القاهرة هي عاصمة مصر، وتقع على ضفاف نهر النيل؟",
-    },
-    {
-        "input": "قال المعلم للطلاب ادرسوا جيدا فالامتحان قريب",
-        "expected": "قال المعلم للطلاب: ادرسوا جيدا، فالامتحان قريب.",
-    },
-]
-
-def count_punct(text):
-    """Count punctuation marks in text."""
-    marks = set('.,;:!?،؛؟')
-    return sum(1 for c in text if c in marks)
-
-def diff_punct(before, after):
-    """Show what punctuation marks were added/removed."""
-    marks = set('.,;:!?،؛؟')
-    before_marks = [(i, c) for i, c in enumerate(before) if c in marks]
-    after_marks = [(i, c) for i, c in enumerate(after) if c in marks]
-    return before_marks, after_marks
-
-def main():
-    print("=" * 80)
-    print("BAYAN PUNCTUATION TRACE — Where do punctuation marks get lost?")
-    print("=" * 80)
-
-    # Load model
-    print("\n[1/2] Loading PuncAra-v1 model...")
-    from nlp.punctuation.punctuation_service import get_punctuation_model, PunctuationChecker
-    punc_checker = get_punctuation_model()
-    print("  ✓ Model loaded\n")
-
-    # Load pipeline tools
-    print("[2/2] Loading pipeline tools...")
-    from app import get_word_diffs
-    from nlp.punctuation.punctuation_rules import validate_punctuation_diff
-    print("  ✓ Tools loaded\n")
-
-    for idx, test in enumerate(TEST_SENTENCES):
-        inp = test["input"]
-        expected = test["expected"]
-        
-        print("─" * 80)
-        print(f"TEST {idx+1}")
-        print(f"  INPUT:    {inp}")
-        print(f"  EXPECTED: {expected}")
-        print(f"  Expected marks: {count_punct(expected)}")
-        print()
-
-        # ─── Stage A: Raw model output (no post-processing) ────────
-        raw_output = punc_checker._fix_punctuation(inp)
-        print(f"  [A] RAW MODEL:     {raw_output}")
-        print(f"      Marks added:   {count_punct(raw_output) - count_punct(inp)}")
-        print()
-
-        # ─── Stage B: After _strip_non_punctuation_changes ─────────
-        stripped = punc_checker._strip_non_punctuation_changes(inp, raw_output)
-        print(f"  [B] STRIP NON-PUNC: {stripped}")
-        if stripped != raw_output:
-            print(f"      ⚠ Changes stripped! Diff from raw:")
-            for w1, w2 in zip(raw_output.split(), stripped.split()):
-                if w1 != w2:
-                    print(f"        '{w1}' → '{w2}'")
-        print(f"      Marks added:   {count_punct(stripped) - count_punct(inp)}")
-        print()
-
-        # ─── Stage C: get_word_diffs ───────────────────────────────
-        # This is what correct() returns after postprocessing
-        from nlp.punctuation.punctuation_rules import arabic_postprocessing
-        final_punc = arabic_postprocessing(stripped)
-        
-        print(f"  [C] FINAL PUNC OUT: {final_punc}")
-        print(f"      Marks added:   {count_punct(final_punc) - count_punct(inp)}")
-        print()
-
-        # ─── Stage D: Word diffs ──────────────────────────────────
-        if final_punc != inp:
-            diffs = get_word_diffs(inp, final_punc)
-            print(f"  [D] WORD DIFFS ({len(diffs)} found):")
-            for d in diffs:
-                orig = d.get('original', '')
-                corr = d.get('correction', '')
-                
-                # Check validate_punctuation_diff
-                is_valid = validate_punctuation_diff(d)
-                
-                # Check alpha match (lock bypass)
-                orig_alpha = re.sub(r'[^\u0600-\u06FFa-zA-Z]', '', orig)
-                corr_alpha = re.sub(r'[^\u0600-\u06FFa-zA-Z]', '', corr)
-                alpha_match = orig_alpha == corr_alpha
-                
-                status_parts = []
-                if not is_valid:
-                    status_parts.append("❌ SAFETY-REJECTED")
-                if not alpha_match:
-                    status_parts.append("❌ LOCK-BLOCKED (alpha differs)")
-                if is_valid and alpha_match:
-                    status_parts.append("✅ WOULD PASS")
-                elif is_valid:
-                    status_parts.append("✅ valid-punc")
-                
-                status = " | ".join(status_parts)
-                print(f"      [{d['start']}:{d['end']}] '{orig}' → '{corr}'  {status}")
-        else:
-            print(f"  [D] NO DIFFS — model returned same text as input!")
-        
-        print()
-
-    # ─── Summary ───────────────────────────────────────────────────
-    print("=" * 80)
-    print("LOSS POINTS SUMMARY")
-    print("=" * 80)
-    print("""
-Where punctuation marks can be lost:
-
-  [A→B] _strip_non_punctuation_changes():
-         If model changes a word's spelling AND adds punctuation,
-         the punctuation transfer logic may fail.
-
-  [B→C] arabic_postprocessing():
-         Typographic cleanup may remove valid marks.
-
-  [C→D] get_word_diffs():
-         Word-level diff may merge/split changes incorrectly.
-
-  [D→E] StageLocker:
-         Locked ranges from spelling/grammar block nearby punctuation.
-         (Now relaxed: pure-punc changes pass through)
-
-  [D→E] validate_punctuation_diff():
-         Safety layer rejects diffs that change Arabic text.
-
-  [E→F] Overlap resolver:
-         Grammar/spelling patches take priority over punctuation.
-
-  [E→F] Patch cap:
-         Max 3 punctuation patches per response.
-""")
-
-
-if __name__ == "__main__":
-    main()