Upload 56 files

api/__init__.py

@@ -0,0 +1 @@
+"""HTTP layer that serves the saved model to the web app."""

api/main.py

@@ -0,0 +1,103 @@
+"""FastAPI app serving the trained model to the web front end.
+
+    cd ml
+    uvicorn api.main:app --reload --port 8000
+"""
+
+import os
+
+from dotenv import load_dotenv
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse
+from pydantic import BaseModel, Field
+
+from api.service import (
+    EmptyAfterCleaning,
+    analyze,
+    describe_model,
+    load_metrics,
+    plot_path,
+)
+
+load_dotenv()
+
+
+def allowed_origins():
+    """Browser origins permitted to call this API."""
+    raw = os.getenv("ASA_CORS_ORIGINS", "http://localhost:3000")
+    return [origin.strip() for origin in raw.split(",") if origin.strip()]
+
+
+app = FastAPI(
+    title="Arabic Sentiment Analysis API",
+    description="Serves the trained TF-IDF + classifier pipeline.",
+    version="1.0.0",
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=allowed_origins(),
+    allow_credentials=False,
+    allow_methods=["GET", "POST"],
+    allow_headers=["*"],
+)
+
+
+class PredictRequest(BaseModel):
+    """One piece of raw Arabic text to classify."""
+    text: str = Field(min_length=1, max_length=5000)
+    convert_emojis: bool = True
+
+
+@app.get("/api/health")
+def health():
+    try:
+        model = describe_model()
+    except FileNotFoundError:
+        raise HTTPException(
+            status_code=503,
+            detail="No saved model found. Run `python run_pipeline.py` first.",
+        )
+    return {"status": "ok", "model": model["name"]}
+
+
+@app.get("/api/model")
+def model_info():
+    """What the served pipeline is, and which score types it can produce."""
+    try:
+        return describe_model()
+    except FileNotFoundError:
+        raise HTTPException(
+            status_code=503,
+            detail="No saved model found. Run `python run_pipeline.py` first.",
+        )
+
+
+@app.get("/api/metrics")
+def metrics():
+    """Test-set scores for all four candidate models, plus the chart manifest."""
+    return load_metrics()
+
+
+@app.get("/api/plots/{name}")
+def plot(name: str):
+    """Serve one of the pipeline's chart PNGs by file name."""
+    path = plot_path(name)
+    if path is None:
+        raise HTTPException(status_code=404, detail=f"No such chart: {name}")
+    return FileResponse(path, media_type="image/png")
+
+
+@app.post("/api/predict")
+def predict(request: PredictRequest):
+    """Clean and classify one piece of text."""
+    try:
+        return analyze(request.text, convert_emojis=request.convert_emojis)
+    except EmptyAfterCleaning as error:
+        raise HTTPException(status_code=422, detail=str(error))
+    except FileNotFoundError:
+        raise HTTPException(
+            status_code=503,
+            detail="No saved model found. Run `python run_pipeline.py` first.",
+        )

api/service.py

@@ -0,0 +1,199 @@
+"""Model loading, prediction and stored-metrics access for the API.
+Only one model is served: whatever is saved as ``artifacts/best_model.joblib``.
+"""
+
+import json
+from functools import lru_cache
+
+from config.constants import MIN_WORDS_PER_SENTENCE, NEGATION_WORDS
+from config.emoji_map import emojis as EMOJI_TO_ARABIC
+from config.paths import ARTIFACTS_DIR, DEFAULT_MODEL_PATH
+from preprocessing.pipeline import clean_text
+from training.model_io import load_model
+from utils.chart_style import PLOTS_SUBDIR
+
+#: Written by ``run_pipeline.py``
+METRICS_PATH = ARTIFACTS_DIR / "metrics.json"
+
+#: Where ``--save-plots`` puts the PNGs.
+PLOTS_DIR = ARTIFACTS_DIR / PLOTS_SUBDIR
+
+#: Stable keys the web app asks for -> the file names the pipeline writes.
+PLOT_FILES = {
+    "sentiment_distribution_raw": "01_sentiment_distribution_raw.png",
+    "sentiment_distribution_balanced": "02_sentiment_distribution_balanced.png",
+    "review_length": "03_review_length.png",
+    "top_tokens": "04_top_tokens.png",
+    "model_comparison": "05_model_comparison.png",
+    "confusion_matrix": "06_confusion_matrix.png",
+    "per_class_metrics": "07_per_class_metrics.png",
+}
+
+
+class EmptyAfterCleaning(ValueError):
+    """Raised when cleaning leaves nothing for the model to classify.
+    Happens when the input has no Arabic content at all - the ``Removing_non_arabic`` step strips it to an empty string.
+    """
+
+
+@lru_cache(maxsize=1)
+def get_model():
+    """Load the saved pipeline once and reuse it for every request."""
+    return load_model(DEFAULT_MODEL_PATH)
+
+
+def describe_model():
+    """Report what the served pipeline actually is."""
+    pipeline = get_model()
+    classifier = pipeline.named_steps["clf"]
+    vectorizer = pipeline.named_steps["vect"]
+
+    return {
+        "name": type(classifier).__name__,
+        "classes": [str(label) for label in classifier.classes_],
+        "vocabulary_size": len(vectorizer.vocabulary_),
+        "supports_probabilities": hasattr(pipeline, "predict_proba"),
+        "supports_margins": hasattr(pipeline, "decision_function"),
+    }
+
+
+def _class_scores(pipeline, cleaned):
+    """Per-class scores for one cleaned string, or ``None`` if unavailable.
+
+    Two different things can come back, and the caller must not conflate them:
+
+    ``probabilities``
+        Real calibrated probabilities summing to 1, from ``predict_proba``.
+        Available for MultinomialNB / LogisticRegression / RandomForest.
+
+    ``margins``
+        Signed distances from ``decision_function``. ``SVC`` has these but no
+        probabilities. They rank the classes but are *not* percentages, so the
+        UI labels them as margins and never renders them as a confidence.
+    """
+    if hasattr(pipeline, "predict_proba"):
+        scores = pipeline.predict_proba([cleaned])[0]
+        return "probabilities", {
+            str(label): float(score)
+            for label, score in zip(pipeline.classes_, scores)
+        }
+
+    if hasattr(pipeline, "decision_function"):
+        scores = pipeline.decision_function([cleaned])[0]
+        return "margins", {
+            str(label): float(score)
+            for label, score in zip(pipeline.classes_, scores)
+        }
+
+    return None, None
+
+
+def _find_negations(text):
+    """Negation particles present in the raw input.
+
+    ``KEEP_NEGATIONS`` is on, so these survive stopword removal - which is the
+    whole point, since dropping them would turn "لا احب" into "احب".
+    """
+    tokens = set(text.split())
+    return sorted(tokens & NEGATION_WORDS)
+
+
+def _find_emojis(text):
+    """Emoji in the raw input paired with the Arabic word they are replaced by."""
+    found = []
+    seen = set()
+    for character in text:
+        if character in EMOJI_TO_ARABIC and character not in seen:
+            seen.add(character)
+            found.append({"emoji": character, "arabic": EMOJI_TO_ARABIC[character]})
+    return found
+
+
+def analyze(text, convert_emojis=True):
+    """Clean ``text``, classify it, and report everything worth showing.
+
+    Args:
+        text: raw user input.
+        convert_emojis: run the emoji substitution steps first. ``True`` matches how the training corpus was built.
+    """
+    pipeline = get_model()
+    original = text.strip()
+    cleaned = clean_text(original, convert_emojis=convert_emojis).strip()
+
+    if not cleaned:
+        raise EmptyAfterCleaning(
+            "Nothing left after cleaning - the input has no Arabic content."
+        )
+
+    label = str(pipeline.predict([cleaned])[0])
+    score_kind, scores = _class_scores(pipeline, cleaned)
+
+    word_count = len(original.split())
+    cleaned_word_count = len(cleaned.split())
+
+    notes = []
+    if word_count < MIN_WORDS_PER_SENTENCE:
+        notes.append(
+            f"Only {word_count} word(s). Reviews shorter than "
+            f"{MIN_WORDS_PER_SENTENCE} words were dropped from the training "
+            f"corpus, so this is outside what the model learned from."
+        )
+    if cleaned_word_count < MIN_WORDS_PER_SENTENCE <= word_count:
+        notes.append(
+            f"Cleaning reduced this to {cleaned_word_count} word(s). Most of the "
+            f"input was stopwords, punctuation or non-Arabic characters."
+        )
+
+    return {
+        "label": label,
+        "model": type(pipeline.named_steps["clf"]).__name__,
+        "original_text": original,
+        "cleaned_text": cleaned,
+        "word_count": word_count,
+        "cleaned_word_count": cleaned_word_count,
+        "score_kind": score_kind,
+        "scores": scores,
+        "negations_found": _find_negations(original),
+        "emojis_found": _find_emojis(original),
+        "notes": notes,
+    }
+
+
+def load_metrics():
+    """Read ``artifacts/metrics.json``, or return an unavailable placeholder.
+    The file is written by ``run_pipeline.py``.
+    """
+    available_plots = {
+        key: name
+        for key, name in PLOT_FILES.items()
+        if (PLOTS_DIR / name).is_file()
+    }
+
+    if not METRICS_PATH.is_file():
+        return {
+            "available": False,
+            "best_model": None,
+            "selected_by": None,
+            "generated_at": None,
+            "dataset": None,
+            "models": [],
+            "plots": available_plots,
+        }
+
+    stored = json.loads(METRICS_PATH.read_text(encoding="utf-8"))
+    stored["available"] = True
+    stored.setdefault("plots", {})
+    # Trust the filesystem over the manifest: charts can be deleted or regenerated without rerunning training.
+    stored["plots"] = available_plots or stored["plots"]
+    return stored
+
+
+def plot_path(name):
+    """Absolute path of a chart PNG, or ``None`` if it is not a known chart.
+    Only names listed in :data:`PLOT_FILES` resolve, so a request cannot walk
+    out of the plots directory.
+    """
+    if name not in PLOT_FILES.values():
+        return None
+    candidate = PLOTS_DIR / name
+    return candidate if candidate.is_file() else None

config/__init__.py

@@ -0,0 +1,46 @@
+"""Shared configuration: paths, constants, emoji maps and stopwords."""
+
+from config.constants import (
+    EMOJI_COUNT_COLUMN,
+    KEEP_NEGATIONS,
+    LABEL_COLUMN,
+    NEGATION_WORDS,
+    RANDOM_STATE,
+    RATING_TO_SENTIMENT,
+    SENTIMENT_PALETTE,
+    TEST_SIZE,
+    TEXT_COLUMN,
+)
+from config.emoji_map import emojis, emoticons_to_emoji
+from config.paths import (
+    ARTIFACTS_DIR,
+    DEFAULT_CLEANED_DATA_PATH,
+    DEFAULT_MODEL_PATH,
+    DEFAULT_RAW_DATA_PATH,
+    DEFAULT_SHARING_DATA_PATH,
+    ORIGINAL_DIR,
+    PROJECT_ROOT,
+)
+from config.stopwords import get_arabic_stopwords
+
+__all__ = [
+    "ARTIFACTS_DIR",
+    "DEFAULT_CLEANED_DATA_PATH",
+    "DEFAULT_MODEL_PATH",
+    "DEFAULT_RAW_DATA_PATH",
+    "DEFAULT_SHARING_DATA_PATH",
+    "EMOJI_COUNT_COLUMN",
+    "KEEP_NEGATIONS",
+    "LABEL_COLUMN",
+    "NEGATION_WORDS",
+    "ORIGINAL_DIR",
+    "PROJECT_ROOT",
+    "RANDOM_STATE",
+    "RATING_TO_SENTIMENT",
+    "SENTIMENT_PALETTE",
+    "TEST_SIZE",
+    "TEXT_COLUMN",
+    "emojis",
+    "emoticons_to_emoji",
+    "get_arabic_stopwords",
+]

config/constants.py

@@ -0,0 +1,37 @@
+TEXT_COLUMN = "text"
+LABEL_COLUMN = "label"
+EMOJI_COUNT_COLUMN = "emoji_count"
+RAW_COLUMNS_TO_DROP = ["Unnamed: 0", "company"]
+RATING_COLUMN = "rating"
+RAW_TEXT_COLUMN = "Feed"
+RAW_TEXT_COLUMN_CANDIDATES = ("Feed", "review_description")
+SENTIMENT_COLUMN = "Sentiment"
+RATING_TO_SENTIMENT = {1: "Positive", 0: "Neutral", -1: "Negative"}
+SENTIMENT_PALETTE = {"Positive": "green", "Neutral": "gray", "Negative": "red"}
+TEST_SIZE = 0.2
+RANDOM_STATE = 42
+MIN_WORDS_PER_SENTENCE = 3
+
+# Arabic negation particles that NLTK lists as stopwords.
+# Removing these destroys polarity - "لا احب" ("I don't like") collapses to "احب" ("I like")
+# They are kept in the text when :data:`KEEP_NEGATIONS` is on.
+NEGATION_WORDS = frozenset({
+    "لا",       # no / not
+    "ما",      # not (also the relative "what")
+    "لم",      # did not
+    "لن",      # will not
+    "غير",     # not / other than
+    "ليس",     # is not
+    "ليست",    # is not (f.)
+    "ليسا",    # are not (dual)
+    "ليسوا",   # are not (pl.)
+    "لست",     # you/I am not
+    "ولا",      # and not / nor
+    "وما",     # and not
+    "دون",     # without
+    "أبدا",    # never / at all
+})
+
+# Keep the words in :data:`NEGATION_WORDS` instead of filtering them out.
+# Changing this changes the training corpus, so the model must be retrained to match.
+KEEP_NEGATIONS = True

config/emoji_map.py

@@ -0,0 +1,288 @@
+"""Emoji / emoticon lookup tables.
+
+``emojis``              -- emoji character -> Arabic word(s).
+``emoticons_to_emoji``  -- ASCII emoticon  -> emoji character (applied first).
+"""
+
+emojis = {
+    "🙂":"يبتسم",
+    "😂":"يضحك",
+    "💔":"قلب حزين",
+    "🙂":"يبتسم",
+    "❤️":"حب",
+    "❤":"حب",
+    "😍":"حب",
+    "😭":"يبكي",
+    "😢":"حزن",
+    "😔":"حزن",
+    "♥":"حب",
+    "💜":"حب",
+    "😅":"يضحك",
+    "🙁":"حزين",
+    "💕":"حب",
+    "💙":"حب",
+    "😞":"حزين",
+    "😊":"سعادة",
+    "👏":"يصفق",
+    "👌":"احسنت",
+    "😴":"ينام",
+    "😀":"يضحك",
+    "😌":"حزين",
+    "🌹":"وردة",
+    "🙈":"حب",
+    "😄":"يضحك",
+    "😐":"محايد",
+    "✌":"منتصر",
+    "✨":"نجمه",
+    "🤔":"تفكير",
+    "😏":"يستهزء",
+    "😒":"يستهزء",
+    "🙄":"ملل",
+    "😕":"عصبية",
+    "😃":"يضحك",
+    "🌸":"وردة",
+    "😓":"حزن",
+    "💞":"حب",
+    "💗":"حب",
+    "😑":"منزعج",
+    "💭":"تفكير",
+    "😎":"ثقة",
+    "💛":"حب",
+    "😩":"حزين",
+    "💪":"عضلات",
+    "👍":"موافق",
+    "🙏🏻":"رجاء طلب",
+    "😳":"مصدوم",
+    "👏🏼":"تصفيق",
+    "🎶":"موسيقي",
+    "🌚":"صمت",
+    "💚":"حب",
+    "🙏":"رجاء طلب",
+    "💘":"حب",
+    "🍃":"سلام",
+    "☺":"يضحك",
+    "🐸":"ضفدع",
+    "😶":"مصدوم",
+    "✌️":"مرح",
+    "✋🏻":"توقف",
+    "😉":"غمزة",
+    "🌷":"حب",
+    "🙃":"مبتسم",
+    "😫":"حزين",
+    "😨":"مصدوم",
+    "🎼 ":"موسيقي",
+    "🍁":"مرح",
+    "🍂":"مرح",
+    "💟":"حب",
+    "😪":"حزن",
+    "😆":"يضحك",
+    "😣":"استياء",
+    "☺️":"حب",
+    "😱":"كارثة",
+    "😁":"يضحك",
+    "😖":"استياء",
+    "🏃🏼":"يجري",
+    "😡":"غضب",
+    "🚶":"يسير",
+    "🤕":"مرض",
+    "‼️":"تعجب",
+    "🕊":"طائر",
+    "👌🏻":"احسنت",
+    "❣":"حب",
+    "🙊":"مصدوم",
+    "💃":"سعادة مرح",
+    "💃🏼":"سعادة مرح",
+    "😜":"مرح",
+    "👊":"ضربة",
+    "😟":"استياء",
+    "💖":"حب",
+    "😥":"حزن",
+    "🎻":"موسيقي",
+    "✒":"يكتب",
+    "🚶🏻":"يسير",
+    "💎":"الماظ",
+    "😷":"وباء مرض",
+    "☝":"واحد",
+    "🚬":"تدخين",
+    "💐" : "ورد",
+    "🌞" : "شمس",
+    "👆" : "الاول",
+    "⚠️" :"تحذير",
+    "🤗" : "احتواء",
+    "✖️": "غلط",
+    "📍"  : "مكان",
+    "👸" : "ملكه",
+    "👑" : "تاج",
+    "✔️" : "صح",
+    "💌": "قلب",
+    "😲" : "مندهش",
+    "💦": "ماء",
+    "🚫" : "خطا",
+    "👏🏻" : "برافو",
+    "🏊" :"يسبح",
+    "👍🏻": "تمام",
+    "⭕️" :"دائره كبيره",
+    "🎷" : "ساكسفون",
+    "👋": "تلويح باليد",
+    "✌🏼": "علامه النصر",
+    "🌝":"مبتسم",
+    "➿"  : "عقده مزدوجه",
+    "💪🏼" : "قوي",
+    "📩":  "تواصل معي",
+    "☕️": "قهوه",
+    "😧" : "قلق و صدمة",
+    "🗨": "رسالة",
+    "❗️" :"تعجب",
+    "🙆🏻": "اشاره موافقه",
+    "👯" :"اخوات",
+    "©" :  "رمز",
+    "👵🏽" :"سيده عجوزه",
+    "🐣": "كتكوت",
+    "🙌": "تشجيع",
+    "🙇": "شخص ينحني",
+    "👐🏽":"ايدي مفتوحه",
+    "👌🏽": "بالظبط",
+    "⁉️" : "استنكار",
+    "⚽️": "كوره",
+    "🕶" :"حب",
+    "🎈" :"بالون",
+    "🎀":    "ورده",
+    "💵":  "فلوس",
+    "😋":  "جائع",
+    "😛":  "يغيظ",
+    "😠":  "غاضب",
+    "✍🏻":  "يكتب",
+    "🌾":  "ارز",
+    "👣":  "اثر قدمين",
+    "❌":"رفض",
+    "🍟":"طعام",
+    "👬":"صداقة",
+    "🐰":"ارنب",
+    "☂":"مطر",
+    "⚜":"مملكة فرنسا",
+    "🐑":"خروف",
+    "🗣":"صوت مرتفع",
+    "👌🏼":"احسنت",
+    "☘":"مرح",
+    "😮":"صدمة",
+    "😦":"قلق",
+    "⭕":"الحق",
+    "✏️":"قلم",
+    "ℹ":"معلومات",
+    "🙍🏻":"رفض",
+    "⚪️":"نضارة نقاء",
+    "🐤":"حزن",
+    "💫":"مرح",
+    "💝":"حب",
+    "🍔":"طعام",
+    "❤︎":"حب",
+    "✈️":"سفر",
+    "🏃🏻‍♀️":"يسير",
+    "🍳":"ذكر",
+    "🎤":"مايك غناء",
+    "🎾":"كره",
+    "🐔":"دجاجة",
+    "🙋":"سؤال",
+    "📮":"بحر",
+    "💉":"دواء",
+    "🙏🏼":"رجاء طلب",
+    "💂🏿 ":"حارس",
+    "🎬":"سينما",
+    "♦️":"مرح",
+    "💡":"قكرة",
+    "‼":"تعجب",
+    "👼":"طفل",
+    "🔑":"مفتاح",
+    "♥️":"حب",
+    "🕋":"كعبة",
+    "🐓":"دجاجة",
+    "💩":"معترض",
+    "👽":"فضائي",
+    "☔️":"مطر",
+    "🍷":"عصير",
+    "🌟":"نجمة",
+    "☁️":"سحب",
+    "👃":"معترض",
+    "🌺":"مرح",
+    "🔪":"سكينة",
+    "♨":"سخونية",
+    "👊🏼":"ضرب",
+    "✏":"قلم",
+    "🚶🏾‍♀️":"يسير",
+    "👊":"ضربة",
+    "◾️":"وقف",
+    "😚":"حب",
+    "🔸":"مرح",
+    "👎🏻":"لا يعجبني",
+    "👊🏽":"ضربة",
+    "😙":"حب",
+    "🎥":"تصوير",
+    "👉":"جذب انتباه",
+    "👏🏽":"يصفق",
+    "💪🏻":"عضلات",
+    "🏴":"اسود",
+    "🔥":"حريق",
+    "😬":"عدم الراحة",
+    "👊🏿":"يضرب",
+    "🌿":"ورقه شجره",
+    "✋🏼":"كف ايد",
+    "👐":"ايدي مفتوحه",
+    "☠️":"وجه مرعب",
+    "🎉":"يهنئ",
+    "🔕" :"صامت",
+    "😿":"وجه حزين",
+    "☹️":"وجه يائس",
+    "😘" :"حب",
+    "😰" :"خوف و حزن",
+    "🌼":"ورده",
+    "💋":  "بوسه",
+    "👇":"لاسفل",
+    "❣️":"حب",
+    "🎧":"سماعات",
+    "📝":"يكتب",
+    "😇":"دايخ",
+    "😈":"رعب",
+    "🏃":"يجري",
+    "✌🏻":"علامه النصر",
+    "🔫":"يضرب",
+    "❗️":"تعجب",
+    "👎":"غير موافق",
+    "🔐":"قفل",
+    "👈":"لليمين",
+    "™":"رمز",
+    "🚶🏽":"يتمشي",
+    "😯":"متفاجأ",
+    "✊":"يد مغلقه",
+    "😻":"اعجاب",
+    "🙉" :"قرد",
+    "👧":"طفله صغيره",
+    "🔴":"دائره حمراء",
+    "💪🏽":"قوه",
+    "💤":"ينام",
+    "👀":"ينظر",
+    "✍🏻":"يكتب",
+    "❄️":"تلج",
+    "💀":"رعب",
+    "😤":"وجه عابس",
+    "🖋":"قلم",
+    "🎩":"كاب",
+    "☕️":"قهوه",
+    "😹":"ضحك",
+    "💓":"حب",
+    "☄️ ":"نار",
+    "👻":"رعب",
+    }
+
+emoticons_to_emoji = {
+    ":)" : "🙂",
+    ":(" : "🙁",
+    "xD" : "😆",
+    ":=(": "😭",
+    ":'(": "😢",
+    ":'‑(": "😢",
+    "XD" : "😂",
+    ":D" : "🙂",
+    "♬" : "موسيقي",
+    "♡" : "❤",
+    "☻"  : "🙂",
+    }

config/paths.py

@@ -0,0 +1,26 @@
+"""Filesystem locations used by the pipeline."""
+
+import os
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+
+ORIGINAL_DIR = PROJECT_ROOT / "original"
+
+DEFAULT_RAW_DATA_PATH = Path(
+    os.getenv("ASA_RAW_DATA_PATH", ORIGINAL_DIR / "original_dataset.xlsx")
+)
+
+ARTIFACTS_DIR = Path(os.getenv("ASA_ARTIFACTS_DIR", PROJECT_ROOT / "artifacts"))
+
+DEFAULT_SHARING_DATA_PATH = Path(
+    os.getenv("ASA_SHARING_DATA_PATH", ARTIFACTS_DIR / "sharing_arabic.xlsx")
+)
+
+DEFAULT_CLEANED_DATA_PATH = Path(
+    os.getenv("ASA_CLEANED_DATA_PATH", ARTIFACTS_DIR / "cleaned.xlsx")
+)
+
+DEFAULT_MODEL_PATH = Path(
+    os.getenv("ASA_MODEL_PATH", ARTIFACTS_DIR / "best_model.joblib")
+)

config/stopwords.py

@@ -0,0 +1,29 @@
+"""Arabic stopword list used by :func:`preprocessing.text_cleaning.remove_stop_words`."""
+
+import nltk
+from nltk.corpus import stopwords
+
+from config.constants import KEEP_NEGATIONS, NEGATION_WORDS
+
+_cache = {}
+
+
+def get_arabic_stopwords(keep_negations=None):
+    """Return the NLTK Arabic stopword list.
+    Downloads the corpus on first use and caches the result per setting.
+    """
+    if keep_negations is None:
+        keep_negations = KEEP_NEGATIONS
+
+    if keep_negations not in _cache:
+        try:
+            nltk.data.find("corpora/stopwords")
+        except LookupError:
+            nltk.download("stopwords")
+
+        words = stopwords.words("arabic")
+        if keep_negations:
+            words = [w for w in words if w not in NEGATION_WORDS]
+        _cache[keep_negations] = words
+
+    return _cache[keep_negations]

inference/__init__.py

@@ -0,0 +1,5 @@
+"""Loading a saved model and predicting on new text."""
+
+from inference.predict import get_default_model, predict_sentiment
+
+__all__ = ["get_default_model", "predict_sentiment"]

inference/predict.py

@@ -0,0 +1,45 @@
+"""Sentiment prediction for new, unseen text."""
+
+import pandas as pd
+
+from config.constants import TEXT_COLUMN
+from config.paths import DEFAULT_MODEL_PATH
+from preprocessing.pipeline import preprocess_dataframe
+from training.model_io import load_model
+
+_default_model = None
+
+
+def get_default_model(path=DEFAULT_MODEL_PATH):
+    """Load (and cache) the saved best model."""
+    global _default_model
+    if _default_model is None:
+        _default_model = load_model(path)
+    return _default_model
+
+
+def predict_sentiment(text, model=None, convert_emojis=True):
+    """Predict the sentiment of a single piece of raw text.
+
+    The text goes through the same cleaning chain the training corpus went
+    through - emoticons to emojis, emojis to Arabic words, remove stopwords,
+    remove non-Arabic characters, normalize, remove numbers, remove
+    hashtags/mentions, remove URLs, remove punctuation, light-stem - and is
+    then handed to the fitted TF-IDF + classifier pipeline.
+    """
+    if model is None:
+        model = get_default_model()
+
+    # Create a DataFrame with the input text
+    text = [text]
+    daf = pd.DataFrame({TEXT_COLUMN: text})
+
+    # Preprocess the text
+    daf = preprocess_dataframe(daf, text_column=TEXT_COLUMN,
+                               convert_emojis=convert_emojis,
+                               drop_small_sentences=False)
+
+    # Make predictions
+    predictions = model.predict(daf[TEXT_COLUMN])
+
+    return predictions

models/__init__.py

@@ -0,0 +1,17 @@
+"""One sub-package per classifier, plus the registry that collects them."""
+
+from models import logistic_regression, naive_bayes, random_forest, svm
+
+__all__ = [
+    "build_models_dict",
+    "logistic_regression",
+    "naive_bayes",
+    "random_forest",
+    "svm",
+]
+
+
+def build_models_dict():
+    from models.registry import build_models_dict as _build_models_dict
+
+    return _build_models_dict()

models/logistic_regression/__init__.py

@@ -0,0 +1,5 @@
+"""Logistic Regression classifier."""
+
+from models.logistic_regression.model import MODEL_NAME, build_model
+
+__all__ = ["MODEL_NAME", "build_model"]

models/logistic_regression/model.py

@@ -0,0 +1,10 @@
+"""Logistic Regression configuration."""
+
+from sklearn.linear_model import LogisticRegression
+
+MODEL_NAME = "LogisticRegression"
+
+
+def build_model():
+    """Return an untrained ``LogisticRegression``."""
+    return LogisticRegression()

models/naive_bayes/__init__.py

@@ -0,0 +1,5 @@
+"""Multinomial Naive Bayes classifier."""
+
+from models.naive_bayes.model import MODEL_NAME, build_model
+
+__all__ = ["MODEL_NAME", "build_model"]

models/naive_bayes/model.py

@@ -0,0 +1,10 @@
+"""Multinomial Naive Bayes configuration."""
+
+from sklearn.naive_bayes import MultinomialNB
+
+MODEL_NAME = "MultinomialNB"
+
+
+def build_model():
+    """Return an untrained ``MultinomialNB``."""
+    return MultinomialNB()

models/random_forest/__init__.py

@@ -0,0 +1,5 @@
+"""Random Forest classifier."""
+
+from models.random_forest.model import MODEL_NAME, build_model
+
+__all__ = ["MODEL_NAME", "build_model"]

models/random_forest/model.py

@@ -0,0 +1,10 @@
+"""Random Forest configuration."""
+
+from sklearn.ensemble import RandomForestClassifier
+
+MODEL_NAME = "RandomForestClassifier"
+
+
+def build_model():
+    """Return an untrained ``RandomForestClassifier``."""
+    return RandomForestClassifier()

models/registry.py

@@ -0,0 +1,10 @@
+"""Registry of the four classifiers compared by the pipeline."""
+
+from models import logistic_regression, naive_bayes, random_forest, svm
+
+MODEL_MODULES = (naive_bayes, random_forest, logistic_regression, svm)
+
+
+def build_models_dict():
+    """Return ``{model_name: untrained estimator}`` for every registered model."""
+    return {module.MODEL_NAME: module.build_model() for module in MODEL_MODULES}

models/svm/__init__.py

@@ -0,0 +1,5 @@
+"""Support Vector classifier."""
+
+from models.svm.model import MODEL_NAME, build_model
+
+__all__ = ["MODEL_NAME", "build_model"]

models/svm/model.py

@@ -0,0 +1,10 @@
+"""Support Vector Classifier configuration."""
+
+from sklearn.svm import SVC
+
+MODEL_NAME = "SVC"
+
+
+def build_model():
+    """Return an untrained ``SVC``."""
+    return SVC()

preprocessing/__init__.py

@@ -0,0 +1,50 @@
+"""Text cleaning / normalization for Arabic sentiment analysis."""
+
+from preprocessing.arabic_normalizer import Arabic_Light_Stemmer, normalizeArabic
+from preprocessing.emoji_handler import (
+    emoji_counter,
+    extract_emoji,
+    remove_emoji,
+    replace_emojis_with_text,
+    replace_emoticon_with_emojis,
+    space_between_emojis,
+)
+from preprocessing.pipeline import (
+    CLEANING_STEPS,
+    EMOJI_STEPS,
+    clean_text,
+    preprocess_dataframe,
+)
+from preprocessing.text_cleaning import (
+    Removing_non_arabic,
+    Removing_numbers,
+    Removing_punctuations,
+    Removing_urls,
+    remove_extra_Space,
+    remove_hashtags_and_mentions,
+    remove_small_sentences,
+    remove_stop_words,
+)
+
+__all__ = [
+    "Arabic_Light_Stemmer",
+    "CLEANING_STEPS",
+    "EMOJI_STEPS",
+    "Removing_non_arabic",
+    "Removing_numbers",
+    "Removing_punctuations",
+    "Removing_urls",
+    "clean_text",
+    "emoji_counter",
+    "extract_emoji",
+    "normalizeArabic",
+    "preprocess_dataframe",
+    "remove_emoji",
+    "remove_extra_Space",
+    "remove_hashtags_and_mentions",
+    "remove_small_sentences",
+    "remove_stop_words",
+    "replace_emojis_with_text",
+    "replace_emoticon_with_emojis",
+    "space_between_emojis",
+]

preprocessing/arabic_normalizer.py

@@ -0,0 +1,51 @@
+"""Arabic-specific normalization and light stemming."""
+
+import re
+import warnings
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", SyntaxWarning)
+    import pyarabic.araby as araby
+    from tashaphyne.stemming import ArabicLightStemmer
+
+
+def normalizeArabic(text):
+    """Unify Arabic letter forms, collapse repetitions and strip diacritics/digits."""
+    text = text.strip()
+    text = re.sub("ى", "ي", text)
+    text = re.sub("ؤ", "ء", text)
+    text = re.sub("ئ", "ء", text)
+    text = re.sub("ة", "ه", text)
+
+    #remove repetetions
+    text = re.sub("[إأٱآا]", "ا", text)
+    text = text.replace('وو', 'و')
+    text = text.replace('يي', 'ي')
+    text = text.replace('ييي', 'ي')
+    text = text.replace('اا', 'ا')
+
+    ## remove extra whitespace
+    text = re.sub(r'\s+', ' ', text)
+
+    # Remove longation
+    text = re.sub(r'(.)\1+', r"\1\1", text)
+
+    # Strip vowels from a text, include Shadda.
+    text = araby.strip_tashkeel(text)
+
+    # Strip diacritics from a text, include harakats and small lettres The striped marks are
+    text = araby.strip_diacritics(text)
+    text = ''.join([i for i in text if not i.isdigit()])
+    return text
+
+
+def Arabic_Light_Stemmer(text):
+    """Light-stem every word of ``text`` with Tashaphyne's ``ArabicLightStemmer``."""
+    # Arabic Light Stemming is a specific type of stemming applied to Arabic words. Stemming is the process
+    # of reducing words to their root or base form, which helps in normalizing text for analysis.
+    Arabic_Stemmer = ArabicLightStemmer()
+
+    # stemming each word
+    text = [Arabic_Stemmer.light_stem(y) for y in text.split()]
+
+    return " ".join(text)

preprocessing/emoji_handler.py

@@ -0,0 +1,67 @@
+"""Emoji and emoticon handling."""
+
+import re
+
+import emoji
+import regex
+
+from config.emoji_map import emojis, emoticons_to_emoji
+
+
+def extract_emoji(text):
+    """Return the list of emoji grapheme clusters contained in ``text``."""
+    emoji_list = []
+    data = regex.findall(r'\X', text)
+    for word in data:
+        if any(emoji.distinct_emoji_list(char) for char in word):
+            emoji_list.append(word)
+
+    return emoji_list
+
+
+def emoji_counter(sentence):
+    """Return the number of emojis in ``sentence`` (used for the EDA table)."""
+    return emoji.emoji_count(sentence)
+
+
+def replace_emoticon_with_emojis(message):
+    """Replace whitespace-delimited ASCII emoticons with their emoji equivalent."""
+    separate_words = message.split(' ')
+    modified_message = ""
+
+    for word in separate_words:
+        modified_message += emoticons_to_emoji.get(word, word) + " "
+
+    return modified_message.strip()  # Remove trailing space
+
+
+def replace_emojis_with_text(message):
+    """Replace each known emoji with the Arabic word(s) describing it."""
+    separate_words = regex.findall(r'\X', message)
+    modified_message = ""
+
+    for word in separate_words:
+        if any(emoji.distinct_emoji_list(char) for char in word):
+            modified_message += " " + emojis.get(word, word) + " "
+        else:
+            modified_message += emojis.get(word, word) + ""
+
+    return modified_message
+
+
+def remove_emoji(string):
+    """Strip emoji characters from ``string`` (not part of the active pipeline)."""
+    emoji_pattern = re.compile("["
+                           u"\U0001F600-\U0001F64F"  # emoticons
+                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
+                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
+                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
+                           u"\U00002702-\U000027B0"
+                           u"\U000024C2-\U0001F251"
+                           "]+", flags=re.UNICODE)
+    return emoji_pattern.sub(r'', string).strip()
+
+
+def space_between_emojis(s):
+    """Pad every emoji with spaces (not part of the active pipeline)."""
+    return ''.join((' '+c+' ') if c in emoji.UNICODE_EMOJI['en'] else c for c in s)

preprocessing/pipeline.py

@@ -0,0 +1,67 @@
+"""The single preprocessing pipeline shared by training and inference.
+
+Order of the steps:
+1. ``replace_emoticon_with_emojis``   (emoji step)
+2. ``replace_emojis_with_text``       (emoji step)
+3. ``remove_stop_words``
+4. ``Removing_non_arabic``
+5. ``normalizeArabic``
+6. ``Removing_numbers``
+7. ``remove_hashtags_and_mentions``
+8. ``Removing_urls``
+9. ``Removing_punctuations``
+10. ``Arabic_Light_Stemmer``
+"""
+
+from config.constants import TEXT_COLUMN
+from preprocessing.arabic_normalizer import Arabic_Light_Stemmer, normalizeArabic
+from preprocessing.emoji_handler import (
+    replace_emojis_with_text,
+    replace_emoticon_with_emojis,
+)
+from preprocessing.text_cleaning import (
+    Removing_non_arabic,
+    Removing_numbers,
+    Removing_punctuations,
+    Removing_urls,
+    remove_hashtags_and_mentions,
+    remove_small_sentences,
+    remove_stop_words,
+)
+
+#: Steps 1-2 - emoticon/emoji substitution.
+EMOJI_STEPS = (
+    replace_emoticon_with_emojis,
+    replace_emojis_with_text,
+)
+
+#: Steps 3-10 - applied to both the training corpus and inference input.
+CLEANING_STEPS = (
+    remove_stop_words,
+    Removing_non_arabic,
+    normalizeArabic,
+    Removing_numbers,
+    remove_hashtags_and_mentions,
+    Removing_urls,
+    Removing_punctuations,
+    Arabic_Light_Stemmer,
+)
+
+
+def clean_text(text, convert_emojis=True):
+    """Run the full cleaning chain on a single string."""
+    steps = (EMOJI_STEPS + CLEANING_STEPS) if convert_emojis else CLEANING_STEPS
+    for step in steps:
+        text = step(text)
+    return text
+
+
+def preprocess_dataframe(df, text_column=TEXT_COLUMN, convert_emojis=True,
+                         drop_small_sentences=True):
+    df[text_column] = df[text_column].apply(
+        lambda text: clean_text(text, convert_emojis=convert_emojis)
+    )
+    if drop_small_sentences:
+        # this function will convert the text which contains one or two words into null value
+        remove_small_sentences(df, text_column=text_column)
+    return df

preprocessing/text_cleaning.py

@@ -0,0 +1,66 @@
+"""Generic text cleaning helpers (stopwords, punctuation, URLs, numbers, ...)."""
+
+import re
+
+import numpy as np
+
+from config.constants import MIN_WORDS_PER_SENTENCE, TEXT_COLUMN
+from config.stopwords import get_arabic_stopwords
+
+
+def remove_stop_words(text):
+    """Drop NLTK Arabic stopwords from ``text``."""
+    arabic_stopwords = get_arabic_stopwords()
+    Text = [i for i in str(text).split() if i not in arabic_stopwords]
+    return " ".join(Text)
+
+
+def Removing_non_arabic(text):
+    """Replace latin letter runs with a single space."""
+    text = re.sub('[A-Za-z]+', ' ', text)
+    return text
+
+
+def Removing_numbers(text):
+    """Drop every digit character."""
+    text = ''.join([i for i in text if not i.isdigit()])
+    return text
+
+
+def Removing_punctuations(text):
+    """Replace punctuation (latin + Arabic) with spaces and squeeze whitespace."""
+    ## Remove punctuations
+    text = re.sub('[%s]' % re.escape(r"""!"#$%&'()*+,،-./:;<=>؟?@[\]^_`{|}~"""), ' ', text)
+    text = text.replace('؛', "", )
+
+    ## remove extra whitespace
+    text = re.sub(r'\s+', ' ', text)
+    text = " ".join(text.split())
+    return text.strip()
+
+
+def Removing_urls(text):
+    """Strip http(s):// and www. URLs."""
+    url_pattern = re.compile(r'https?://\S+|www\.\S+')
+    return url_pattern.sub(r'', text)
+
+
+def remove_extra_Space(text):
+    """Collapse repeated whitespace into single spaces."""
+    text = re.sub(r'\s+', ' ', text)
+    return " ".join(text.split())
+
+
+def remove_hashtags_and_mentions(text):
+    """Strip ``@mentions`` and ``#hashtags`` written with latin characters."""
+    text = re.sub("@[A-Za-z0-9_]+", "", text)
+    text = re.sub("#[A-Za-z0-9_]+", "", text)
+    return text
+
+
+def remove_small_sentences(df, text_column=TEXT_COLUMN):
+    """Turn texts shorter than 3 words into NaN, in place."""
+    text_position = df.columns.get_loc(text_column)
+    for i in range(len(df)):
+        if len(df[text_column].iloc[i].split()) < MIN_WORDS_PER_SENTENCE:
+            df.iloc[i, text_position] = np.nan

utils/__init__.py

@@ -0,0 +1,22 @@
+"""Cross-cutting helpers: charts and exploratory statistics."""
+
+from utils.emoji_stats import add_emoji_count, most_common_emojis
+from utils.visualization import (
+    plot_confusion_matrix,
+    plot_model_comparison,
+    plot_per_class_metrics,
+    plot_review_length_distribution,
+    plot_sentiment_distribution,
+    plot_top_tokens,
+)
+
+__all__ = [
+    "add_emoji_count",
+    "most_common_emojis",
+    "plot_confusion_matrix",
+    "plot_model_comparison",
+    "plot_per_class_metrics",
+    "plot_review_length_distribution",
+    "plot_sentiment_distribution",
+    "plot_top_tokens",
+]