Update app.py

app.py

@@ -1,57 +1,45 @@
 import gradio as gr
 import torch
 import sentencepiece as spm
-import os
 from transformers import RobertaForTokenClassification
+from huggingface_hub import hf_hub_download
+import csv
+import io
 
-# ─── Load model & tokenizer ───────────────────────
-MODEL_PATH = "hellosindh/sindhi-bert-ner"
-SP_MODEL   = "sindhi_bpe_32k.model"
+MODEL_ID = "hellosindh/sindhi-bert-ner"
 
 print("Loading model...", flush=True)
-model = RobertaForTokenClassification.from_pretrained(
-    MODEL_PATH
-)
+model = RobertaForTokenClassification.from_pretrained(MODEL_ID)
 model.eval()
 
 print("Loading tokenizer...", flush=True)
+sp_path = hf_hub_download(repo_id=MODEL_ID, filename="sindhi_bpe_32k.model")
 sp = spm.SentencePieceProcessor()
-sp.Load(SP_MODEL)
+sp.Load(sp_path)
+print("✅ Ready!", flush=True)
 
-# ─── Tag config ───────────────────────────────────
 ID2TAG = model.config.id2label
-
 BOS_ID = 2
 EOS_ID = 3
 
-# Entity colors for highlighting
-COLORS = {
-    "PERSON":       "#FF6B6B",
-    "LOCATION":     "#4ECDC4",
-    "ORGANIZATION": "#45B7D1",
-    "DATE_TIME":    "#96CEB4",
-    "EVENT":        "#FFEAA7",
-    "LITERARY_WORK":"#DDA0DD",
-    "PROFESSION":   "#98D8C8",
-    "TITLE":        "#F7DC6F",
-    "LANGUAGE":     "#BB8FCE",
-    "FIELD":        "#85C1E9",
-    "LAW":          "#F0B27A",
-    "GROUP":        "#82E0AA",
-    "MISC":         "#BDC3C7",
+ENTITY_CONFIG = {
+    "PERSON":        {"color": "#c084fc", "bg": "rgba(192,132,252,0.15)", "sindhi": "ماڻهو"},
+    "LOCATION":      {"color": "#818cf8", "bg": "rgba(129,140,248,0.15)", "sindhi": "جڳهه"},
+    "ORGANIZATION":  {"color": "#38bdf8", "bg": "rgba(56,189,248,0.15)",  "sindhi": "ادارو"},
+    "DATE_TIME":     {"color": "#34d399", "bg": "rgba(52,211,153,0.15)",  "sindhi": "تاريخ"},
+    "EVENT":         {"color": "#fbbf24", "bg": "rgba(251,191,36,0.15)",  "sindhi": "واقعو"},
+    "TITLE":         {"color": "#fb923c", "bg": "rgba(251,146,60,0.15)",  "sindhi": "لقب"},
 }
 
-# ─── Prediction function ──────────────────────────
+
 def predict_ner(sentence):
     if not sentence.strip():
-        return "", []
-    
-    words = sentence.split()
-    
-    # Tokenize
+        return _empty_html(), _empty_summary(), "", None
+
+    words     = sentence.split()
     input_ids = [BOS_ID]
-    word_map  = [-1]  # maps token → word index
-    
+    word_map  = [-1]
+
     for i, word in enumerate(words):
         subwords = sp.EncodeAsIds(word)
         if not subwords:
@@ -59,149 +47,517 @@ def predict_ner(sentence):
         for j, sw in enumerate(subwords):
             input_ids.append(sw)
             word_map.append(i if j == 0 else -1)
-    
+
     input_ids.append(EOS_ID)
     word_map.append(-1)
-    
-    # Run model
+
     tensor = torch.tensor([input_ids])
     with torch.no_grad():
         logits = model(tensor).logits[0]
-    
+
+    probs = torch.softmax(logits, dim=-1)
     preds = torch.argmax(logits, dim=-1).tolist()
-    
-    # Collect word-level predictions
+    conf  = probs.max(dim=-1).values.tolist()
+
     word_tags = {}
+    word_conf = {}
     for pos, (pred, wid) in enumerate(zip(preds, word_map)):
         if wid >= 0:
             word_tags[wid] = ID2TAG[pred]
-    
-    # ─── Build highlighted HTML ───────────────────
-    html_parts = []
+            word_conf[wid] = conf[pos]
+
     entities   = []
-    
+    html_words = []
     i = 0
+
     while i < len(words):
         tag = word_tags.get(i, "O")
-        
+
         if tag.startswith("B-"):
-            entity_type = tag[2:]
+            etype        = tag[2:]
             entity_words = [words[i]]
-            
-            # Collect I- continuation tokens
+            scores       = [word_conf.get(i, 0)]
+
             j = i + 1
             while j < len(words):
-                next_tag = word_tags.get(j, "O")
-                if next_tag == f"I-{entity_type}":
+                if word_tags.get(j, "O") == f"I-{etype}":
                     entity_words.append(words[j])
+                    scores.append(word_conf.get(j, 0))
                     j += 1
                 else:
                     break
-            
+
             entity_text = " ".join(entity_words)
-            color = COLORS.get(entity_type, "#BDC3C7")
-            
-            html_parts.append(
-                f'<mark style="background:{color}; '
-                f'padding:2px 6px; border-radius:4px; '
-                f'margin:2px; font-weight:bold;" '
-                f'title="{entity_type}">'
-                f'{entity_text} '
-                f'<span style="font-size:0.75em; '
-                f'opacity:0.8;">[{entity_type}]</span>'
-                f'</mark>'
+            avg_score   = sum(scores) / len(scores)
+            cfg         = ENTITY_CONFIG.get(etype, ENTITY_CONFIG["MISC"])
+
+            html_words.append(
+                f'<span style="'
+                f'background:{cfg["bg"]};'
+                f'border:1px solid {cfg["color"]}50;'
+                f'color:#f1f5f9;'
+                f'padding:3px 10px 3px 6px;'
+                f'border-radius:8px;'
+                f'margin:2px;'
+                f'display:inline-block;'
+                f'font-weight:500;">'
+                f'<span style="'
+                f'background:{cfg["color"]};'
+                f'color:#0a0a1a;'
+                f'font-size:0.6em;font-weight:800;'
+                f'padding:1px 6px;border-radius:4px;'
+                f'margin-left:6px;vertical-align:middle;'
+                f'letter-spacing:0.3px;">'
+                f'{cfg["sindhi"]}</span>'
+                f'{entity_text}'
+                f'</span>'
             )
-            
-            entities.append((entity_text, entity_type))
+            entities.append({
+                "text":   entity_text,
+                "type":   etype,
+                "sindhi": cfg["sindhi"],
+                "score":  avg_score,
+                "color":  cfg["color"],
+            })
             i = j
-        
         else:
-            html_parts.append(words[i])
+            html_words.append(
+                f'<span style="color:#cbd5e1;padding:2px 3px;">{words[i]}</span>'
+            )
             i += 1
-    
-    html = '<p dir="rtl" style="font-size:1.2em; ' \
-           'line-height:2.5em; text-align:right;">' + \
-           " ".join(html_parts) + "</p>"
-    
-    # Build entity table
-    table = []
-    for text, etype in entities:
-        table.append([text, etype])
-    
-    return html, table
-
-# ─── Example sentences ────────────────────────────
-examples = [
-    ["شيخ اياز شڪارپور ۾ پيدا ٿيو"],
-    ["سنڌ يونيورسٽي حيدرآباد ۾ آھي"],
-    ["پاڪستان ڏکڻ ايشيا ۾ آھي"],
-    ["ڊاڪٽر محمد علي 1990ع ۾ سنڌ آيو"],
-]
-
-# ─── Gradio Interface ─────────────────────────────
-with gr.Blocks(
-    theme=gr.themes.Soft(),
-    title="Sindhi NER"
-) as demo:
-    
-    gr.Markdown("""
-    # 🏷️ Sindhi Named Entity Recognizer
-    ### سنڌي نالن جي سڃاڻپ جو اوزار
-    
-    First Sindhi NER model — trained on 22,777 annotated sentences!
-    
-    **Recognizes:** Person · Location · Organization · 
-    Date/Time · Event · Literary Work · and 15 more types
-    """)
-    
+
+    highlighted = f"""
+    <div style="
+        background:linear-gradient(135deg,#1a0533 0%,#0f0f2e 100%);
+        border:1px solid #7c3aed30;
+        border-radius:16px;
+        padding:24px 28px;
+        font-size:1.2em;
+        line-height:3.2em;
+        direction:rtl;
+        text-align:right;
+        font-family:'Noto Nastaliq Urdu',Georgia,serif;
+        min-height:90px;
+        box-shadow:inset 0 0 60px #7c3aed08;">
+        {" ".join(html_words)}
+    </div>
+    """
+
+    summary  = _build_summary(entities)
+    conf_html = _build_confidence(entities)
+    csv_file  = _build_csv(entities)
+
+    return highlighted, summary, conf_html, csv_file
+
+
+def _empty_html():
+    return """
+    <div style="
+        background:linear-gradient(135deg,#1a0533,#0f0f2e);
+        border:1px solid #7c3aed20;
+        border-radius:16px;
+        padding:40px;
+        text-align:center;
+        min-height:90px;
+        display:flex;align-items:center;justify-content:center;">
+        <span style="color:#4c1d95;font-size:1em;font-family:Georgia,serif;">
+           ڪو بہ سنڌي جملو لکو ✦
+        </span>
+    </div>
+    """
+
+def _empty_summary():
+    return """
+    <div style="
+        background:#1a0533;
+        border:1px solid #7c3aed20;
+        border-radius:16px;
+        padding:24px;
+        text-align:center;
+        color:#4c1d95;
+        font-size:0.9em;">
+        اعتماد جوڳا نتيجا نہ مليا
+    </div>
+    """
+
+def _build_summary(entities):
+    if not entities:
+        return _empty_summary()
+
+    from collections import Counter
+    counts = Counter(e["type"] for e in entities)
+    total  = len(entities)
+
+    cards = ""
+    for etype, cnt in sorted(counts.items(), key=lambda x: -x[1]):
+        cfg = ENTITY_CONFIG.get(etype, ENTITY_CONFIG["MISC"])
+        cards += f"""
+        <div style="
+            background:{cfg['bg']};
+            border:1px solid {cfg['color']}40;
+            border-radius:10px;
+            padding:10px 14px;
+            display:flex;
+            justify-content:space-between;
+            align-items:center;
+            margin-bottom:8px;
+            direction:rtl;">
+            <span style="color:{cfg['color']};font-weight:600;font-size:0.88em;">
+                {cfg['sindhi']}
+            </span>
+            <span style="
+                background:{cfg['color']};
+                color:#0a0a1a;
+                font-weight:800;
+                border-radius:20px;
+                padding:1px 10px;
+                font-size:0.82em;
+                min-width:24px;text-align:center;">
+                {cnt}
+            </span>
+        </div>
+        """
+
+    return f"""
+    <div style="
+        background:linear-gradient(135deg,#1a0533,#0f0f2e);
+        border:1px solid #7c3aed30;
+        border-radius:16px;
+        padding:18px 16px;">
+        <div style="
+            color:#c084fc;font-weight:700;font-size:0.88em;
+            margin-bottom:12px;padding-bottom:10px;
+            border-bottom:1px solid #7c3aed25;
+            direction:rtl;text-align:right;
+            letter-spacing:0.3px;">
+            مجموعي:
+            <span style="color:#f1f5f9;font-size:1.1em;">{total}</span>
+            سڃاڻپ
+        </div>
+        <div>{cards}</div>
+    </div>
+    """
+
+def _build_confidence(entities):
+    if not entities:
+        return ""
+
+    bars = ""
+    for ent in entities:
+        cfg   = ENTITY_CONFIG.get(ent["type"], ENTITY_CONFIG["MISC"])
+        pct   = int(ent["score"] * 100)
+        width = pct
+
+        bars += f"""
+        <div style="margin-bottom:16px;direction:rtl;">
+            <div style="
+                display:flex;justify-content:space-between;
+                align-items:center;margin-bottom:6px;">
+                <span style="color:#e2e8f0;font-size:0.9em;font-weight:500;
+                    font-family:Georgia,serif;">
+                    {ent['text']}
+                </span>
+                <div style="display:flex;gap:8px;align-items:center;">
+                    <span style="
+                        background:{cfg['color']}18;
+                        border:1px solid {cfg['color']}40;
+                        color:{cfg['color']};
+                        font-size:0.68em;padding:2px 8px;
+                        border-radius:4px;font-weight:700;">
+                        {ent['sindhi']}
+                    </span>
+                    <span style="color:{cfg['color']};
+                        font-weight:800;font-size:0.9em;
+                        font-family:monospace;">
+                        {pct}%
+                    </span>
+                </div>
+            </div>
+            <div style="
+                background:#1e1040;
+                border-radius:999px;height:5px;overflow:hidden;">
+                <div style="
+                    width:{width}%;height:100%;
+                    background:linear-gradient(90deg,
+                        {cfg['color']}60,{cfg['color']});
+                    border-radius:999px;">
+                </div>
+            </div>
+        </div>
+        """
+
+    return f"""
+    <div style="
+        background:linear-gradient(135deg,#1a0533,#0f0f2e);
+        border:1px solid #7c3aed30;
+        border-radius:16px;
+        padding:20px 18px;
+        margin-top:4px;">
+        <div style="
+            color:#c084fc;font-weight:700;font-size:0.88em;
+            margin-bottom:16px;padding-bottom:10px;
+            border-bottom:1px solid #7c3aed25;
+            direction:rtl;text-align:right;">
+            اعتماد
+        </div>
+        {bars}
+    </div>
+    """
+
+def _build_csv(entities):
+    if not entities:
+        return None
+    output = io.StringIO()
+    writer = csv.writer(output)
+    writer.writerow(["Entity", "Type", "Sindhi Type", "Confidence"])
+    for ent in entities:
+        writer.writerow([
+            ent["text"], ent["type"],
+            ent["sindhi"], f"{ent['score']*100:.1f}%"
+        ])
+    path = "/tmp/sindhi_ner.csv"
+    with open(path, "w", encoding="utf-8-sig", newline="") as f:
+        f.write(output.getvalue())
+    return path
+
+
+CSS = """
+@import url('https://fonts.googleapis.com/css2?family=Noto+Nastaliq+Urdu:wght@400;700&family=Space+Mono:wght@700&family=Outfit:wght@300;400;600;700;800&display=swap');
+
+*, body, .gradio-container {
+    font-family: 'Outfit', sans-serif !important;
+}
+body, .gradio-container {
+    background: #08081a !important;
+}
+.gradio-container {
+    max-width: 980px !important;
+    margin: 0 auto !important;
+    padding: 16px !important;
+}
+label > span {
+    color: #9333ea !important;
+    font-size: 0.82em !important;
+    font-weight: 700 !important;
+    letter-spacing: 0.8px !important;
+    text-transform: uppercase !important;
+}
+textarea, input[type="text"] {
+    background: #130825 !important;
+    border: 1px solid #6d28d960 !important;
+    border-radius: 14px !important;
+    color: #e2e8f0 !important;
+    font-size: 1.1em !important;
+    direction: rtl !important;
+    font-family: Georgia, 'Noto Nastaliq Urdu', serif !important;
+    caret-color: #c084fc !important;
+}
+textarea:focus {
+    border-color: #c084fc !important;
+    box-shadow: 0 0 0 3px #7c3aed15 !important;
+    outline: none !important;
+}
+button.primary {
+    background: linear-gradient(135deg, #6d28d9, #9333ea, #c084fc) !important;
+    background-size: 200% auto !important;
+    border: none !important;
+    border-radius: 12px !important;
+    color: #fff !important;
+    font-weight: 800 !important;
+    font-size: 0.95em !important;
+    letter-spacing: 0.5px !important;
+    transition: all 0.3s ease !important;
+    padding: 13px !important;
+}
+button.primary:hover {
+    background-position: right center !important;
+    box-shadow: 0 6px 24px #7c3aed50 !important;
+    transform: translateY(-1px) !important;
+}
+button.secondary {
+    background: #130825 !important;
+    border: 1px solid #6d28d940 !important;
+    border-radius: 12px !important;
+    color: #7c3aed !important;
+    font-weight: 600 !important;
+    transition: all 0.2s !important;
+}
+button.secondary:hover {
+    border-color: #c084fc !important;
+    color: #c084fc !important;
+}
+.examples-holder, .examples table {
+    background: #130825 !important;
+    border: 1px solid #6d28d930 !important;
+    border-radius: 12px !important;
+}
+.examples table td {
+    color: #94a3b8 !important;
+    font-family: Georgia, serif !important;
+}
+.examples table tr:hover td {
+    color: #c084fc !important;
+    background: #1a0533 !important;
+}
+.gap-4 { gap: 12px !important; }
+::-webkit-scrollbar { width: 5px; }
+::-webkit-scrollbar-track { background: #08081a; }
+::-webkit-scrollbar-thumb { background: #6d28d9; border-radius: 3px; }
+"""
+
+HEADER = """
+<div style="
+    background:linear-gradient(135deg,#1a0533 0%,#0f0f2e 60%,#160a2e 100%);
+    border:1px solid #7c3aed25;
+    border-radius:20px;
+    padding:36px 28px 28px;
+    margin-bottom:20px;
+    text-align:center;
+    position:relative;overflow:hidden;">
+    <div style="
+        position:absolute;top:0;left:0;right:0;bottom:0;
+        background:radial-gradient(ellipse at 50% 0%,#7c3aed12 0%,transparent 65%);
+        pointer-events:none;"></div>
+    <div style="position:relative;">
+        <div style="font-size:3em;margin-bottom:8px;line-height:1;">🏷️</div>
+        <h1 style="
+            color:#f1f5f9;
+            font-size:2em;font-weight:800;
+            margin:0 0 4px;
+            letter-spacing:-1px;
+            text-shadow:0 0 40px #7c3aed50;">
+            سنڌي اسمن جي سڃاڻپ
+        </h1>
+        <p style="
+            font-family:'Space Mono',monospace;
+            color:#6d28d9;font-size:0.72em;
+            letter-spacing:3px;margin:0 0 18px;">
+            SINDHI NAMED ENTITY RECOGNITION
+        </p>
+        <div style="display:flex;justify-content:center;gap:10px;flex-wrap:wrap;">
+            <span style="
+                background:#7c3aed15;border:1px solid #7c3aed35;
+                color:#a855f7;padding:5px 14px;border-radius:20px;
+                font-size:0.75em;font-weight:600;">
+                ✦ 22,777 جملا
+            </span>
+            <span style="
+                background:#7c3aed15;border:1px solid #7c3aed35;
+                color:#a855f7;padding:5px 14px;border-radius:20px;
+                font-size:0.75em;font-weight:600;">
+                ✦ 6 قسم
+            </span>
+            <span style="
+                background:#7c3aed15;border:1px solid #7c3aed35;
+                color:#a855f7;padding:5px 14px;border-radius:20px;
+                font-size:0.75em;font-weight:600;">
+                ✦ 125M پيراميٽر
+            </span>
+            <span style="
+                background:#7c3aed15;border:1px solid #7c3aed35;
+                color:#a855f7;padding:5px 14px;border-radius:20px;
+                font-size:0.75em;font-weight:600;">
+                ✦ سنڌي BERT
+            </span>
+        </div>
+    </div>
+</div>
+"""
+
+LEGEND = """
+<div style="
+    background:linear-gradient(135deg,#1a0533,#0f0f2e);
+    border:1px solid #7c3aed20;
+    border-radius:14px;
+    padding:16px 18px;
+    margin-top:4px;">
+    <div style="
+        color:#9333ea;font-weight:700;font-size:0.78em;
+        letter-spacing:1px;margin-bottom:12px;
+        direction:rtl;text-align:right;">
+       اسمن جي نشاندھي
+    </div>
+    <div style="display:flex;flex-wrap:wrap;gap:8px;direction:rtl;">
+        <span style="background:rgba(192,132,252,0.15);border:1px solid #c084fc40;color:#c084fc;padding:3px 10px;border-radius:6px;font-size:0.75em;font-weight:600;">ماڻهو</span>
+        <span style="background:rgba(129,140,248,0.15);border:1px solid #818cf840;color:#818cf8;padding:3px 10px;border-radius:6px;font-size:0.75em;font-weight:600;">جڳهه</span>
+        <span style="background:rgba(56,189,248,0.15);border:1px solid #38bdf840;color:#38bdf8;padding:3px 10px;border-radius:6px;font-size:0.75em;font-weight:600;">ادارو</span>
+        <span style="background:rgba(52,211,153,0.15);border:1px solid #34d39940;color:#34d399;padding:3px 10px;border-radius:6px;font-size:0.75em;font-weight:600;">تاريخ</span>
+        <span style="background:rgba(251,191,36,0.15);border:1px solid #fbbf2440;color:#fbbf24;padding:3px 10px;border-radius:6px;font-size:0.75em;font-weight:600;">واقعو</span>
+        <span style="background:rgba(251,146,60,0.15);border:1px solid #fb923c40;color:#fb923c;padding:3px 10px;border-radius:6px;font-size:0.75em;font-weight:600;">لقب</span>
+        </div>
+</div>
+"""
+
+
+with gr.Blocks(css=CSS, title="سنڌي NER") as demo:
+
+    gr.HTML(HEADER)
+
     with gr.Row():
-        with gr.Column():
-            text_input = gr.Textbox(
-                label="سنڌي جملو لکو (Enter Sindhi text)",
-                placeholder="شيخ اياز شڪارپور ۾ پيدا ٿيو",
-                lines=3,
+        with gr.Column(scale=3):
+            inp = gr.Textbox(
+                label="سنڌي جملو لکو",
+                placeholder="شيخ اياز شڪارپور ۾ پيدا ٿيو...",
+                lines=4,
                 rtl=True
             )
-            submit_btn = gr.Button(
-                "🔍 Entities ڳوليو",
-                variant="primary"
-            )
-    
-    with gr.Row():
-        highlighted = gr.HTML(
-            label="Highlighted Entities"
-        )
-    
+            with gr.Row():
+                btn = gr.Button("🔍  ڳوليو", variant="primary")
+                gr.ClearButton(value="🗑️  مٽايو", components=[inp], variant="secondary")
+
+        with gr.Column(scale=2):
+            summary_out = gr.HTML(value=_empty_summary())
+
+    gr.HTML("<div style='height:4px'></div>")
+
+    highlighted_out = gr.HTML(
+        label="نتيجا(و)",
+        value=_empty_html()
+    )
+
+    conf_out = gr.HTML()
+
     with gr.Row():
-        entity_table = gr.Dataframe(
-            headers=["Entity", "Type"],
-            label="Entities Found",
-            wrap=True
+        csv_out = gr.File(
+            label="📥 ڊائونلوڊ ڪريو (CSV)",
+            file_types=[".csv"],
+            interactive=False
         )
-    
-    # Color legend
-    gr.Markdown("""
-    ### Legend
-    🔴 Person &nbsp; 🟦 Location &nbsp; 
-    🔵 Organization &nbsp; 🟢 Date/Time &nbsp;
-    🟡 Event &nbsp; 🟣 Literary Work
-    """)
-    
+
+    gr.HTML(LEGEND)
+
     gr.Examples(
-        examples=examples,
-        inputs=text_input
+        label="مثالي جملا",
+        examples=[
+            ["شيخ اياز شڪارپور ۾ پيدا ٿيو"],
+            ["يونيورسٽي آف سنڌ حيدرآباد ۾ آھي"],
+            ["سيد مراد علي شاھ سنڌ جو وڏو وزير آھي، سندس تعلق پاڪستان پيپلز پارٽي سان آھي"],
+            ["پاڪستان ۽ ڀارت جي ويڙھ 2025ع ۾ لڳي."],
+            ["ڊاڪٽر نبي بخش بلوچ 16 ڊسمبر 1917ع تي سنجھوري ۾ پيدا ٿيو"],
+            ["بينظير ڀٽو پاڪستان جي پھرين عورت وزيراعظم هئي"],
+        ],
+        inputs=inp
     )
-    
-    submit_btn.click(
+
+    gr.HTML("""
+    <div style="
+        text-align:center;padding:20px 0 8px;
+        font-family:'Space Mono',monospace;
+        color:#3b0764;font-size:0.72em;
+        letter-spacing:1.5px;">
+        hellosindh · sindhi-bert-ner · MIT License
+    </div>
+    """)
+
+    btn.click(
         fn=predict_ner,
-        inputs=text_input,
-        outputs=[highlighted, entity_table]
+        inputs=inp,
+        outputs=[highlighted_out, summary_out, conf_out, csv_out]
     )
-    
-    text_input.submit(
+    inp.submit(
         fn=predict_ner,
-        inputs=text_input,
-        outputs=[highlighted, entity_table]
+        inputs=inp,
+        outputs=[highlighted_out, summary_out, conf_out, csv_out]
     )
 
 demo.launch()