Upload 2 files

app.py

@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+import emoji
+import re
+import nltk
+import torch
+import numpy as np
+import joblib
+import pandas as pd
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+
+# ── 1. Setup & Pre-processing Environment ──
+nltk.download('stopwords', quiet=True)
+from nltk.corpus import stopwords
+arabic_stopwords = set(stopwords.words('arabic'))
+
+def clean_text(text):
+    text = re.sub(r'http\S+|www\S+|https\S+', '', text, flags=re.MULTILINE)
+    text = re.sub(r'[@#]', '', text) 
+    text = re.sub(r'\d+', '', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+def remove_emojis(text):
+    if not isinstance(text, str):
+        return text
+    return emoji.replace_emoji(text, replace='')
+
+def remove_arabic_punctuation(text):
+    if not isinstance(text, str):
+        return text
+    arabic_punct = (
+        r'[\u0600-\u0605\u060C\u060D\u061B\u061C\u061D\u061E\u061F'
+        r'\u0640\u066A-\u066D\u06D4\u200c\u200d\u200e\u200f'
+        r'\ufeff\u202a-\u202e،؟؛«»]'
+    )
+    text = re.sub(arabic_punct, ' ', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+def normalize_arabic_characters(text):
+    if not isinstance(text, str):
+        text = str(text)
+    text = re.sub(r'[أإآ]', 'ا', text)   
+    text = text.replace('ى', 'ي')                    
+    text = text.replace('ة', 'ه')                    
+    text = re.sub(r'[\u064B-\u065F\u0670]', '', text) 
+    return text
+
+def remove_repeated_chars(text):
+    if not isinstance(text, str):
+        return text
+    return re.sub(r'(.)\1{2,}', r'\1\1', text)
+
+def remove_repeated_words(text):
+    if not isinstance(text, str):
+        return text
+    return re.sub(r'\b(\w+)(\s+\1){1,}\b', r'\1', text)
+
+def tokenize_text(text):
+    return text.split() if text else []
+
+def remove_stopwords(tokens, stopwords_set=arabic_stopwords):
+    return [t for t in tokens if t not in stopwords_set]
+
+def preprocess_arabic_text(text):
+    if pd.isna(text) or not isinstance(text, str):
+        return ''
+    text = clean_text(text)
+    text = remove_emojis(text)
+    text = remove_arabic_punctuation(text)
+    text = normalize_arabic_characters(text)
+    text = remove_repeated_chars(text)
+    text = remove_repeated_words(text)
+    tokens = tokenize_text(text)
+    tokens = remove_stopwords(tokens)
+    return ' '.join(tokens)
+
+# ── 2. Load Model ──
+print("Loading model and tokenizer...")
+REPO_ID = "mahmoudmohammad/marbertv2-multilabel-dialect"
+tokenizer = AutoTokenizer.from_pretrained(REPO_ID)
+model = AutoModelForSequenceClassification.from_pretrained(REPO_ID)
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+model.to(device)
+model.eval()
+
+try:
+    mlb = joblib.load('mlb_dialects.pkl')
+    class_names = list(mlb.classes_)
+except FileNotFoundError:
+    class_names = ['Bahraini', 'Egyptian', 'Emirati', 'Jordanian', 'Lebanese', 'MSA', 'Palestinian', 'Qatari', 'Saudi', 'Syrian']
+
+
+# ── 3. Prediction Pipeline ──
+def predict_dialects(text, threshold):
+    cleaned_text = preprocess_arabic_text(text)
+
+    inputs = tokenizer(
+        cleaned_text,
+        return_tensors="pt",
+        truncation=True,
+        max_length=256
+    )
+    inputs = {k: v.to(device) for k, v in inputs.items()}
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+        logits = outputs.logits
+
+    probs = torch.sigmoid(logits).squeeze().cpu().numpy()
+    predictions = (probs > threshold).astype(int)
+
+    predicted_dialects = []
+    predicted_probs = {}
+
+    for i, pred in enumerate(predictions):
+        dialect = class_names[i]
+        predicted_probs[dialect] = round(float(probs[i]), 4)
+        if pred == 1:
+            predicted_dialects.append(dialect)
+
+    # Fallback to single highest
+    if len(predicted_dialects) == 0:
+        max_idx = np.argmax(probs)
+        predicted_dialects.append(class_names[max_idx])
+
+    dialects_out = ", ".join(predicted_dialects)
+    return dialects_out, predicted_probs, cleaned_text
+
+
+# ── 4. Gradio UI with Enforced Dark Mode ──
+
+# The following script is a standard hack to forcefully enable dark mode on app load.
+dark_mode_js = """
+function() {
+    document.body.classList.add('dark');
+}
+"""
+
+with gr.Blocks(theme=gr.themes.Base(), title="Arabic Multi-Dialect Analyzer") as demo:
+    gr.Markdown(
+        """
+        # 🌙 Multi-Label Arabic Dialect Inference
+        Identify overlapping dialects in modern Arabic text seamlessly.
+        *Powered by MARBERTv2*
+        """
+    )
+    
+    with gr.Row():
+        with gr.Column():
+            text_input = gr.Textbox(
+                lines=5, 
+                label="Arabic Text Input", 
+                placeholder="أدخل النص العربي هنا...", 
+                rtl=True
+            )
+            threshold_slider = gr.Slider(
+                minimum=0.1, 
+                maximum=0.9, 
+                step=0.01, 
+                value=0.45, 
+                label="Confidence Threshold",
+                info="Determines minimum probability needed to label a dialect."
+            )
+            submit_btn = gr.Button("Analyze Text", variant="primary")
+            
+        with gr.Column():
+            dialects_output = gr.Textbox(label="Predicted Dialect(s)")
+            prob_output = gr.Label(label="Confidence Probabilities", num_top_classes=10)
+            clean_text_output = gr.Textbox(label="Text After Pre-Processing", rtl=True)
+    
+    # Optional Examples for fast-testing
+    gr.Examples(
+        examples=[
+            ["شلونك اليوم؟ شو عم تعمل؟", 0.45],
+            ["انا رايح الشغل بدري علشان عندي شغل كتير.", 0.45]
+        ],
+        inputs=[text_input, threshold_slider]
+    )
+    
+    # Map button click to backend logic
+    submit_btn.click(
+        fn=predict_dialects, 
+        inputs=[text_input, threshold_slider], 
+        outputs=[dialects_output, prob_output, clean_text_output]
+    )
+
+    # Inject JS at initialization to Force Dark Theme
+    demo.load(None, None, None, js=dark_mode_js)
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,8 @@
+torch
+transformers
+gradio
+emoji
+nltk
+pandas
+joblib
+numpy