Deploy DeBERTa MCQ solver with Gradio

README.md

@@ -1,14 +1,23 @@
 ---
-title: Smart Mcq Solver
-emoji: 🏃
-colorFrom: red
-colorTo: indigo
+title: Smart MCQ Solver (DeBERTa)
+emoji: 🧠
+colorFrom: blue
+colorTo: purple
 sdk: gradio
-sdk_version: 6.22.0
-python_version: '3.12'
+sdk_version: "4.44.0"
 app_file: app.py
 pinned: false
-license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🧠 Smart MCQ Solver — DeBERTa-v3-small
+
+Answers 5-option multiple-choice questions using a fine-tuned DeBERTa-v3-small
+model — the best individual model from the Smart MCQ Solver project
+(0.7544 MAP@3 on the Kaggle leaderboard).
+
+## How it works
+
+Each (question, option) pair is scored independently as "correct" vs "wrong"
+using a pretrained DeBERTa-v3-small backbone with a custom classification head
+on the `[CLS]` token. The 5 option scores per question are ranked to produce
+a top-3 answer.

app.py

@@ -0,0 +1,68 @@
+"""
+app.py
+======
+Gradio app for HF Spaces — deploys the best single model (DeBERTa-v3-small,
+0.7544 MAP@3 on the Kaggle leaderboard) as a standalone MCQ solver.
+"""
+
+import gradio as gr
+
+from src.models.deberta import DeBERTaModel
+
+_model = None
+
+
+def get_model():
+    global _model
+    if _model is None:
+        _model = DeBERTaModel().load()
+    return _model
+
+
+def predict(prompt, opt_a, opt_b, opt_c, opt_d, opt_e):
+    if not prompt or not prompt.strip():
+        return "⚠️ Please enter a question.", {}
+
+    options = [opt_a, opt_b, opt_c, opt_d, opt_e]
+    if any(not (o or "").strip() for o in options):
+        return "⚠️ Please fill in all 5 options (A-E).", {}
+
+    model = get_model()
+    proba = model.predict_proba_single(prompt, options)
+    top3 = model.predict_top3_single(prompt, options)
+
+    option_text = {"A": opt_a, "B": opt_b, "C": opt_c, "D": opt_d, "E": opt_e}
+    medal = ["🥇", "🥈", "🥉"]
+    answer_text = "\n".join(
+        f"{medal[i]} **{letter}**: {option_text[letter]}" for i, letter in enumerate(top3)
+    )
+
+    confidences = {letter: float(score) for letter, score in zip("ABCDE", proba)}
+    return answer_text, confidences
+
+
+demo = gr.Interface(
+    fn=predict,
+    inputs=[
+        gr.Textbox(label="Question", lines=3,
+                    placeholder="e.g. What is the powerhouse of the cell?"),
+        gr.Textbox(label="Option A"),
+        gr.Textbox(label="Option B"),
+        gr.Textbox(label="Option C"),
+        gr.Textbox(label="Option D"),
+        gr.Textbox(label="Option E"),
+    ],
+    outputs=[
+        gr.Markdown(label="Top-3 Answer"),
+        gr.Label(label="Confidence per option", num_top_classes=5),
+    ],
+    title="🧠 Smart MCQ Solver — DeBERTa-v3-small",
+    description=(
+        "Answers 5-option multiple-choice questions using a fine-tuned "
+        "**DeBERTa-v3-small** model — the best individual model in this "
+        "project (0.7544 MAP@3 on the Kaggle leaderboard)."
+    ),
+)
+
+if __name__ == "__main__":
+    demo.launch()

outputs/checkpoints/deberta_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17c75824be1016c3b66f52108306f91efa99824fd2a9b10cb8a7ca36c6c9a609
+size 565267989

requirements.txt

@@ -0,0 +1,5 @@
+gradio==4.44.0
+torch==2.3.1
+transformers==4.40.0
+sentencepiece==0.2.0
+numpy==1.26.4

src/__init__.py

@@ -0,0 +1,5 @@
+"""Smart MCQ Solver — production src package."""
+
+from .predict import predict
+
+__all__ = ["predict"]

src/config.py

@@ -0,0 +1,121 @@
+"""
+config.py
+=========
+Single source of truth for every hyperparameter, path, and constant used
+across the Smart MCQ Solver project. Every notebook (01_eda -> 06_ensemble)
+used these exact values during training, so inference MUST use the same
+ones or predictions will not match the trained checkpoints.
+"""
+
+from pathlib import Path
+
+# ---------------------------------------------------------------------------
+# Paths
+# ---------------------------------------------------------------------------
+BASE_DIR       = Path(__file__).resolve().parent.parent
+DATA_DIR       = BASE_DIR / "data"
+OUTPUT_DIR     = BASE_DIR / "outputs"
+CHECKPOINT_DIR = OUTPUT_DIR / "checkpoints"
+PREDICTIONS_DIR = OUTPUT_DIR / "predictions"
+LOGS_DIR       = OUTPUT_DIR / "logs"
+
+for d in (CHECKPOINT_DIR, PREDICTIONS_DIR, LOGS_DIR):
+    d.mkdir(parents=True, exist_ok=True)
+
+# ---------------------------------------------------------------------------
+# Answer label maps (used identically in all 4 model notebooks)
+# ---------------------------------------------------------------------------
+ANSWER_MAP  = {"A": 0, "B": 1, "C": 2, "D": 3, "E": 4}
+REVERSE_MAP = {v: k for k, v in ANSWER_MAP.items()}
+OPTION_COLS = list("ABCDE")
+
+# ---------------------------------------------------------------------------
+# Reproducibility
+# ---------------------------------------------------------------------------
+RANDOM_STATE = 42
+
+# ---------------------------------------------------------------------------
+# 1. TF-IDF + Logistic Regression baseline  (02_baseline.ipynb)
+#    Best strategy in the notebook was picked dynamically (max val MAP@3);
+#    all three text-builder strategies are implemented in preprocessing.py.
+#    Standalone leaderboard score: 0.751
+# ---------------------------------------------------------------------------
+TFIDF_CFG = {
+    "text_strategy": "v3_labeled",   # change if a different variant was your best
+    "max_features": 15000,
+    "min_df": 2,
+    "max_df": 0.9,
+    "ngram_range": (1, 3),
+    "lr_C": 3.0,
+    "model_path": CHECKPOINT_DIR / "tfidf_best_lr_model.pkl",
+    "vectorizer_path": CHECKPOINT_DIR / "tfidf_best_vectorizer.pkl",
+}
+
+# ---------------------------------------------------------------------------
+# 2. LSTM from scratch (03_lstm.ipynb)
+#    Standalone leaderboard score: 0.7543
+# ---------------------------------------------------------------------------
+LSTM_CFG = {
+    "vocab_size": 10000,
+    "max_len": 256,
+    "embed_dim": 128,
+    "hidden_dim": 256,
+    "num_layers": 2,
+    "dropout": 0.4,
+    "num_classes": 5,
+    "checkpoint_path": CHECKPOINT_DIR / "lstm_best.pt",
+    "tokenizer_path": CHECKPOINT_DIR / "lstm_tokenizer.pkl",
+}
+
+# ---------------------------------------------------------------------------
+# 3. DeBERTa-v3-small option scorer (04_DeBERTa.ipynb)
+#    Standalone leaderboard score: 0.7547 (best single model)
+# ---------------------------------------------------------------------------
+DEBERTA_CFG = {
+    "model_name": "microsoft/deberta-v3-small",
+    "max_len": 256,
+    "dropout": 0.3,
+    "checkpoint_path": CHECKPOINT_DIR / "deberta_best.pt",
+}
+
+# ---------------------------------------------------------------------------
+# 4. RoBERTa multiple-choice (05_RoBERTa.ipynb)
+#    Standalone leaderboard score: 0.75436
+# ---------------------------------------------------------------------------
+ROBERTA_CFG = {
+    "model_name": "roberta-base",
+    "max_len": 128,
+    "checkpoint_path": CHECKPOINT_DIR / "roberta_best.pt",
+}
+
+# ---------------------------------------------------------------------------
+# 5. Ensemble weights (06_ensemble.ipynb) — weighted rank ensemble
+#    Tuned empirically against the leaderboard. Must sum to 1.0.
+#    Final ensemble leaderboard score: 0.76018 (best overall)
+# ---------------------------------------------------------------------------
+ENSEMBLE_WEIGHTS = {
+    "tfidf": 0.15,
+    "lstm": 0.20,
+    "deberta": 0.40,
+    "roberta": 0.25,
+}
+
+assert abs(sum(ENSEMBLE_WEIGHTS.values()) - 1.0) < 1e-9, "Ensemble weights must sum to 1.0!"
+
+# ---------------------------------------------------------------------------
+# Individual model standalone scores (for reference / README / app.py display)
+# ---------------------------------------------------------------------------
+LEADERBOARD_SCORES = {
+    "tfidf": 0.7510,
+    "lstm": 0.7543,
+    "roberta": 0.75436,
+    "deberta": 0.7547,
+    "ensemble": 0.76018,
+}
+
+
+# WANDB SETTINGS
+WANDB_PROJECT = "23f2003236-t22026"
+WANDB_ENTITY = "23f2003236"  
+
+

src/dataset.py

@@ -0,0 +1,108 @@
+"""
+dataset.py
+==========
+PyTorch Dataset classes for batch training/evaluation (used if you ever
+retrain a model, not needed for single-question inference in predict.py,
+which calls the tokenizer directly for speed/simplicity).
+"""
+
+from typing import List
+
+import pandas as pd
+import torch
+from torch.utils.data import Dataset
+
+from .config import ANSWER_MAP, OPTION_COLS
+from .preprocessing import build_lstm_text
+
+
+class MCQDataset(Dataset):
+    """LSTM dataset (03_lstm.ipynb) — one combined text per question."""
+
+    def __init__(self, df: pd.DataFrame, tokenizer, max_len: int, is_test: bool = False):
+        self.df = df.reset_index(drop=True)
+        self.tokenizer = tokenizer
+        self.max_len = max_len
+        self.is_test = is_test
+
+    def __len__(self):
+        return len(self.df)
+
+    def __getitem__(self, idx):
+        row = self.df.iloc[idx]
+        text = build_lstm_text(row["prompt"], [row[c] for c in OPTION_COLS])
+
+        ids = self.tokenizer.encode(text)
+        ids = self.tokenizer.pad_or_truncate(ids, self.max_len)
+        id_tensor = torch.tensor(ids, dtype=torch.long)
+
+        if self.is_test:
+            return id_tensor
+
+        label = ANSWER_MAP[row["answer"]]
+        return id_tensor, torch.tensor(label, dtype=torch.long)
+
+
+class OptionDataset(Dataset):
+    """DeBERTa dataset (04_DeBERTa.ipynb) — expects a pre-expanded pairs_df
+    (see preprocessing.expand_pairs), one row per (question, option) pair."""
+
+    def __init__(self, pairs_df: pd.DataFrame, tokenizer, max_len: int, is_test: bool = False):
+        self.df = pairs_df.reset_index(drop=True)
+        self.tok = tokenizer
+        self.max_len = max_len
+        self.is_test = is_test
+
+    def __len__(self):
+        return len(self.df)
+
+    def __getitem__(self, idx):
+        row = self.df.iloc[idx]
+        enc = self.tok(
+            row["text"],
+            max_length=self.max_len,
+            truncation=True,
+            padding="max_length",
+            return_tensors="pt",
+        )
+        item = {
+            "input_ids": enc["input_ids"].squeeze(0),
+            "attention_mask": enc["attention_mask"].squeeze(0),
+        }
+        if "token_type_ids" in enc:
+            item["token_type_ids"] = enc["token_type_ids"].squeeze(0)
+        if not self.is_test:
+            item["label"] = torch.tensor(row["label"], dtype=torch.long)
+        return item
+
+
+class MCQDatasetHF(Dataset):
+    """RoBERTa dataset (05_RoBERTa.ipynb) — AutoModelForMultipleChoice format:
+    5 stacked (prompt, option) encodings per question."""
+
+    def __init__(self, df: pd.DataFrame, tokenizer, max_len: int, is_test: bool = False):
+        self.df = df.reset_index(drop=True)
+        self.tok = tokenizer
+        self.max_len = max_len
+        self.is_test = is_test
+
+    def __len__(self):
+        return len(self.df)
+
+    def __getitem__(self, idx):
+        row = self.df.iloc[idx]
+        prompt = row["prompt"]
+        options = [row[c] for c in OPTION_COLS]
+
+        enc = self.tok(
+            [prompt] * 5, options,
+            max_length=self.max_len, truncation=True, padding="max_length",
+            return_tensors="pt",
+        )
+        item = {
+            "input_ids": enc["input_ids"],        # (5, max_len)
+            "attention_mask": enc["attention_mask"],  # (5, max_len)
+        }
+        if not self.is_test:
+            item["labels"] = torch.tensor(ANSWER_MAP[row["answer"]], dtype=torch.long)
+        return item

src/ensemble.py

@@ -0,0 +1,64 @@
+"""
+ensemble.py
+===========
+Weighted rank ensemble (06_ensemble.ipynb) — combines TF-IDF, LSTM, DeBERTa,
+and RoBERTa top-3 predictions into one final top-3, using reciprocal-rank
+scoring weighted by each model's trustworthiness.
+
+For every model's top-3 list:
+    1st choice -> weight * 1.0
+    2nd choice -> weight * 0.5
+    3rd choice -> weight * 0.333
+
+Scores are summed across all 4 models; the 3 options with the highest
+combined score become the final prediction, in order.
+"""
+
+from collections import defaultdict
+from typing import Dict, List
+
+from .config import ENSEMBLE_WEIGHTS
+
+
+def rank_ensemble(top3_by_model: Dict[str, List[str]],
+                   weights: Dict[str, float] = None) -> List[str]:
+    """
+    Args:
+        top3_by_model: e.g. {
+            "tfidf":   ["B", "A", "D"],
+            "lstm":    ["B", "D", "C"],
+            "deberta": ["B", "A", "C"],
+            "roberta": ["A", "B", "D"],
+        }
+        weights: model_name -> weight (defaults to config.ENSEMBLE_WEIGHTS)
+
+    Returns:
+        Final top-3 option letters, e.g. ["B", "A", "D"]
+    """
+    weights = weights or ENSEMBLE_WEIGHTS
+    scores = defaultdict(float)
+
+    for model_name, top3 in top3_by_model.items():
+        w = weights.get(model_name, 0.0)
+        for rank, option in enumerate(top3):
+            scores[option] += w * (1.0 / (rank + 1))
+
+    best_3 = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:3]
+    return [option for option, _ in best_3]
+
+
+def rank_ensemble_with_scores(top3_by_model: Dict[str, List[str]],
+                               weights: Dict[str, float] = None):
+    """Same as rank_ensemble but also returns the raw per-option score dict,
+    useful for the Streamlit app to show a confidence breakdown."""
+    weights = weights or ENSEMBLE_WEIGHTS
+    scores = defaultdict(float)
+
+    for model_name, top3 in top3_by_model.items():
+        w = weights.get(model_name, 0.0)
+        for rank, option in enumerate(top3):
+            scores[option] += w * (1.0 / (rank + 1))
+
+    ranked = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+    top3_final = [option for option, _ in ranked[:3]]
+    return top3_final, dict(ranked)

src/inference.py

@@ -0,0 +1,68 @@
+"""
+inference.py
+============
+Loads all 4 trained models (TF-IDF, LSTM, DeBERTa, RoBERTa) and exposes a
+single interface to get each model's top-3 prediction for one question.
+Model loading is lazy + cached, so app.py only pays the load cost once.
+"""
+
+from typing import Dict, List
+
+from .models import TFIDFModel, LSTMModel, DeBERTaModel, RoBERTaModel
+from .utils import get_logger
+
+logger = get_logger(__name__)
+
+# Module-level cache so Streamlit doesn't reload models on every rerun
+_MODELS = {}
+
+
+def load_all_models(which: List[str] = None) -> Dict[str, object]:
+    """
+    Load (or return cached) model instances.
+
+    Args:
+        which: subset of ["tfidf", "lstm", "deberta", "roberta"] to load.
+               Defaults to all four.
+    """
+    which = which or ["tfidf", "lstm", "deberta", "roberta"]
+
+    loaders = {
+        "tfidf": TFIDFModel,
+        "lstm": LSTMModel,
+        "deberta": DeBERTaModel,
+        "roberta": RoBERTaModel,
+    }
+
+    for name in which:
+        if name not in _MODELS:
+            logger.info(f"Loading {name} model...")
+            _MODELS[name] = loaders[name]().load()
+            logger.info(f"{name} model loaded.")
+
+    return {name: _MODELS[name] for name in which}
+
+
+def predict_single_model(model_name: str, prompt: str, options: List[str]) -> List[str]:
+    """Run one model on one question, return its top-3 option letters."""
+    models = load_all_models([model_name])
+    return models[model_name].predict_top3_single(prompt, options)
+
+
+def predict_all_models(prompt: str, options: List[str]) -> Dict[str, List[str]]:
+    """
+    Run all 4 models on one question.
+
+    Returns:
+        {
+            "tfidf":   ["B", "A", "D"],
+            "lstm":    ["B", "D", "C"],
+            "deberta": ["B", "A", "C"],
+            "roberta": ["A", "B", "D"],
+        }
+    """
+    models = load_all_models()
+    return {
+        name: model.predict_top3_single(prompt, options)
+        for name, model in models.items()
+    }

src/metrics.py

@@ -0,0 +1,68 @@
+"""
+metrics.py
+==========
+MAP@3 (Mean Average Precision @ 3) — the competition metric used identically
+across all 6 notebooks:
+
+    Position of correct answer  |  Score
+    ----------------------------|-------
+    1st                         |  1.00
+    2nd                         |  0.50
+    3rd                         |  0.33
+    Not in top-3                |  0.00
+"""
+
+from typing import Sequence
+import numpy as np
+
+try:
+    import torch
+except ImportError:  # torch not required for the TF-IDF-only path
+    torch = None
+
+
+def map_at_3(y_true: Sequence[int], y_proba: np.ndarray) -> float:
+    """
+    MAP@3 from a numpy probability matrix.
+
+    Args:
+        y_true:  1D array/list of true integer class labels (0-4)
+        y_proba: 2D array (n_samples, n_classes) of predicted probabilities
+
+    Returns:
+        float MAP@3 score
+    """
+    scores = []
+    for i, true in enumerate(y_true):
+        top3 = np.argsort(y_proba[i])[-3:][::-1]
+        hit = np.where(top3 == true)[0]
+        score = 1.0 / (hit[0] + 1) if len(hit) else 0.0
+        scores.append(score)
+    return float(np.mean(scores))
+
+
+def map_at_3_from_logits(logits, labels) -> float:
+    """
+    Same metric, but takes raw torch logits + integer label tensor
+    (used in the LSTM and RoBERTa training loops).
+    """
+    if torch is None:
+        raise ImportError("torch is required for map_at_3_from_logits")
+    probs = torch.softmax(logits, dim=1).cpu().numpy()
+    labels = labels.cpu().numpy()
+    return map_at_3(labels, probs)
+
+
+def validate_submission_format(predictions: Sequence[str]) -> int:
+    """
+    Sanity check used at the end of every notebook: every prediction string
+    must be exactly 3 space-separated letters from A-E.
+
+    Returns the number of malformed rows (should be 0).
+    """
+    errors = sum(
+        1
+        for p in predictions
+        if len(str(p).split()) != 3 or not all(c in "ABCDE" for c in str(p).split())
+    )
+    return errors

src/models/__init__.py

@@ -0,0 +1,11 @@
+from .tfidf import TFIDFModel
+from .lstm import LSTMModel, LSTMClassifier, MCQTokenizer
+from .deberta import DeBERTaModel, DeBERTaOptionScorer
+from .roberta import RoBERTaModel
+
+__all__ = [
+    "TFIDFModel",
+    "LSTMModel", "LSTMClassifier", "MCQTokenizer",
+    "DeBERTaModel", "DeBERTaOptionScorer",
+    "RoBERTaModel",
+]

src/models/deberta.py

@@ -0,0 +1,106 @@
+"""
+models/deberta.py
+==================
+DeBERTa-v3-small binary option scorer (04_DeBERTa.ipynb).
+
+Approach: each (prompt, option) pair is scored independently as
+"wrong" vs "correct". At inference we softmax and use P(correct)
+to rank the 5 options for a question.
+
+Architecture:
+    DeBERTa backbone (pretrained) -> [CLS] embedding (768-dim)
+        -> Dropout(0.3) -> Linear(768 -> 2)  [wrong, correct]
+"""
+
+from pathlib import Path
+from typing import List
+
+import numpy as np
+import torch
+import torch.nn as nn
+from transformers import AutoTokenizer, AutoModel
+
+from ..config import DEBERTA_CFG
+from ..preprocessing import build_deberta_pairs_single
+from ..utils import get_device
+
+
+class DeBERTaOptionScorer(nn.Module):
+    """Exact copy of the notebook's model class."""
+
+    def __init__(self, model_name: str, dropout: float = 0.3):
+        super().__init__()
+        # Force fp32: some HF checkpoints ship fp16 weights by default under
+        # torch_dtype="auto", which then mismatches our fp32 classification
+        # head (RuntimeError: mat1 and mat2 must have the same dtype).
+        self.backbone = AutoModel.from_pretrained(model_name, torch_dtype=torch.float32)
+        hidden_size = self.backbone.config.hidden_size
+        self.drop = nn.Dropout(dropout)
+        self.fc = nn.Linear(hidden_size, 2)  # binary: wrong vs correct
+
+    def forward(self, input_ids, attention_mask, token_type_ids=None):
+        kwargs = dict(input_ids=input_ids, attention_mask=attention_mask)
+        if token_type_ids is not None:
+            kwargs["token_type_ids"] = token_type_ids
+
+        out = self.backbone(**kwargs)
+        cls = out.last_hidden_state[:, 0, :]   # [CLS] token representation
+        return self.fc(self.drop(cls))          # (B, 2)
+
+
+class DeBERTaModel:
+    """Convenience wrapper: loads tokenizer + checkpoint, scores all 5 options."""
+
+    def __init__(self, checkpoint_path: Path = None, device=None):
+        self.checkpoint_path = Path(checkpoint_path or DEBERTA_CFG["checkpoint_path"])
+        self.device = device or get_device()
+        self.tokenizer = None
+        self.model: DeBERTaOptionScorer = None
+
+    def load(self) -> "DeBERTaModel":
+        self.tokenizer = AutoTokenizer.from_pretrained(DEBERTA_CFG["model_name"])
+        self.model = DeBERTaOptionScorer(
+            DEBERTA_CFG["model_name"], dropout=DEBERTA_CFG["dropout"]
+        ).to(self.device)
+        self.model.load_state_dict(torch.load(self.checkpoint_path, map_location=self.device))
+        self.model.eval()
+        return self
+
+    @torch.no_grad()
+    def predict_proba_single(self, prompt: str, options: List[str]) -> np.ndarray:
+        """
+        Returns a (5,) array of P(correct) scores, one per option (A-E order).
+        NOTE: these 5 scores are independent sigmoid-like scores, not a
+        joint softmax over the 5 options (unlike RoBERTa) — that mirrors
+        exactly how the notebook ranks options at inference time.
+        """
+        if self.model is None:
+            self.load()
+
+        pair_texts = build_deberta_pairs_single(prompt, options)
+        scores = []
+        for text in pair_texts:
+            enc = self.tokenizer(
+                text,
+                max_length=DEBERTA_CFG["max_len"],
+                truncation=True,
+                padding="max_length",
+                return_tensors="pt",
+            )
+            input_ids = enc["input_ids"].to(self.device)
+            attention_mask = enc["attention_mask"].to(self.device)
+            token_type_ids = enc.get("token_type_ids")
+            if token_type_ids is not None:
+                token_type_ids = token_type_ids.to(self.device)
+
+            logits = self.model(input_ids, attention_mask, token_type_ids)
+            p_correct = torch.softmax(logits, dim=1)[:, 1].cpu().numpy()[0]
+            scores.append(p_correct)
+
+        return np.array(scores)
+
+    def predict_top3_single(self, prompt: str, options: List[str]) -> List[str]:
+        from ..config import REVERSE_MAP
+        scores = self.predict_proba_single(prompt, options)
+        top3_idx = np.argsort(scores)[-3:][::-1]
+        return [REVERSE_MAP[i] for i in top3_idx]

src/models/lstm.py

@@ -0,0 +1,145 @@
+"""
+models/lstm.py
+==============
+Custom word-level tokenizer + Bidirectional LSTM classifier
+(03_lstm.ipynb — the nn.LSTM-backed production version, not the
+from-scratch educational one, since that's what was actually trained).
+
+Architecture:
+    [Input IDs] -> Embedding -> BiLSTM x2 -> Dropout -> Linear -> 5 logits
+"""
+
+from collections import Counter
+from pathlib import Path
+from typing import List
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+from ..config import LSTM_CFG
+from ..preprocessing import build_lstm_text
+from ..utils import load_pickle, get_device
+
+
+class MCQTokenizer:
+    """Simple whitespace word-level tokenizer with a fixed-size vocabulary."""
+
+    PAD_TOKEN = "<PAD>"
+    UNK_TOKEN = "<UNK>"
+
+    def __init__(self, vocab_size: int = 10000):
+        self.vocab_size = vocab_size
+        self.word2idx = {self.PAD_TOKEN: 0, self.UNK_TOKEN: 1}
+        self.idx2word = {0: self.PAD_TOKEN, 1: self.UNK_TOKEN}
+        self.vocab_built = False
+
+    def tokenize(self, text: str) -> List[str]:
+        return str(text).lower().split()
+
+    def build_vocab(self, texts: List[str]):
+        counter = Counter()
+        for text in texts:
+            counter.update(self.tokenize(text))
+
+        most_common = counter.most_common(self.vocab_size - 2)
+        for idx, (word, _) in enumerate(most_common, start=2):
+            self.word2idx[word] = idx
+            self.idx2word[idx] = word
+        self.vocab_built = True
+
+    def encode(self, text: str) -> List[int]:
+        words = self.tokenize(text)
+        return [self.word2idx.get(w, 1) for w in words]
+
+    def pad_or_truncate(self, ids: List[int], max_len: int) -> List[int]:
+        if len(ids) >= max_len:
+            return ids[:max_len]
+        return ids + [0] * (max_len - len(ids))
+
+
+class LSTMClassifier(nn.Module):
+    """Embedding -> BiLSTM x2 -> Dropout -> Linear head. Exact copy from the notebook."""
+
+    def __init__(self, vocab_size, embed_dim, hidden_dim, num_layers,
+                 num_classes, dropout, pad_idx: int = 0):
+        super().__init__()
+
+        self.embedding = nn.Embedding(
+            num_embeddings=vocab_size, embedding_dim=embed_dim, padding_idx=pad_idx
+        )
+        self.lstm = nn.LSTM(
+            input_size=embed_dim,
+            hidden_size=hidden_dim,
+            num_layers=num_layers,
+            batch_first=True,
+            bidirectional=True,
+            dropout=dropout if num_layers > 1 else 0.0,
+        )
+        self.dropout = nn.Dropout(dropout)
+        self.fc = nn.Linear(hidden_dim * 2, num_classes)
+
+    def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
+        embedded = self.embedding(input_ids)
+        embedded = self.dropout(embedded)
+
+        output, (hidden, cell) = self.lstm(embedded)
+
+        forward_hidden = hidden[-2]   # last layer, forward direction
+        backward_hidden = hidden[-1]  # last layer, backward direction
+        combined = torch.cat([forward_hidden, backward_hidden], dim=1)
+
+        out = self.dropout(combined)
+        logits = self.fc(out)
+        return logits
+
+
+class LSTMModel:
+    """Convenience wrapper: loads tokenizer + trained checkpoint, predicts on raw text."""
+
+    def __init__(self, checkpoint_path: Path = None, tokenizer_path: Path = None, device=None):
+        self.checkpoint_path = Path(checkpoint_path or LSTM_CFG["checkpoint_path"])
+        self.tokenizer_path = Path(tokenizer_path or LSTM_CFG["tokenizer_path"])
+        self.device = device or get_device()
+        self.tokenizer: MCQTokenizer = None
+        self.model: LSTMClassifier = None
+
+    def load(self) -> "LSTMModel":
+        # The tokenizer was pickled from inside a notebook, where MCQTokenizer
+        # lived in the __main__ module. When unpickling from any other script
+        # (like this one, or app.py), Python looks for MCQTokenizer in
+        # __main__ and fails unless we register it there first.
+        import sys
+        sys.modules["__main__"].MCQTokenizer = MCQTokenizer
+
+        self.tokenizer = load_pickle(self.tokenizer_path)
+        self.model = LSTMClassifier(
+            vocab_size=LSTM_CFG["vocab_size"],
+            embed_dim=LSTM_CFG["embed_dim"],
+            hidden_dim=LSTM_CFG["hidden_dim"],
+            num_layers=LSTM_CFG["num_layers"],
+            num_classes=LSTM_CFG["num_classes"],
+            dropout=LSTM_CFG["dropout"],
+        ).to(self.device)
+        self.model.load_state_dict(torch.load(self.checkpoint_path, map_location=self.device))
+        self.model.eval()
+        return self
+
+    @torch.no_grad()
+    def predict_proba_single(self, prompt: str, options: List[str]) -> np.ndarray:
+        if self.model is None:
+            self.load()
+        text = build_lstm_text(prompt, options)
+        ids = self.tokenizer.encode(text)
+        ids = self.tokenizer.pad_or_truncate(ids, LSTM_CFG["max_len"])
+        id_tensor = torch.tensor(ids, dtype=torch.long).unsqueeze(0).to(self.device)
+
+        logits = self.model(id_tensor)
+        proba = torch.softmax(logits, dim=1).cpu().numpy()[0]
+        return proba
+
+    def predict_top3_single(self, prompt: str, options: List[str]) -> List[str]:
+        from ..config import REVERSE_MAP
+        proba = self.predict_proba_single(prompt, options)
+        top3_idx = np.argsort(proba)[-3:][::-1]
+        return [REVERSE_MAP[i] for i in top3_idx]

src/models/roberta.py

@@ -0,0 +1,68 @@
+"""
+models/roberta.py
+==================
+RoBERTa via AutoModelForMultipleChoice (05_RoBERTa.ipynb).
+
+Unlike DeBERTa (5 independent binary passes), RoBERTa sees all 5 options
+together and produces one joint softmax over the 5 choices — the
+multiple-choice head is built into `AutoModelForMultipleChoice`, no
+custom classification head needed.
+"""
+
+from pathlib import Path
+from typing import List
+
+import numpy as np
+import torch
+from transformers import AutoTokenizer, AutoModelForMultipleChoice
+
+from ..config import ROBERTA_CFG
+from ..preprocessing import build_roberta_inputs_single
+from ..utils import get_device
+
+
+class RoBERTaModel:
+    """Convenience wrapper: loads tokenizer + checkpoint, predicts option probs."""
+
+    def __init__(self, checkpoint_path: Path = None, device=None):
+        self.checkpoint_path = Path(checkpoint_path or ROBERTA_CFG["checkpoint_path"])
+        self.device = device or get_device()
+        self.tokenizer = None
+        self.model: AutoModelForMultipleChoice = None
+
+    def load(self) -> "RoBERTaModel":
+        self.tokenizer = AutoTokenizer.from_pretrained(ROBERTA_CFG["model_name"])
+        self.model = AutoModelForMultipleChoice.from_pretrained(
+            ROBERTA_CFG["model_name"]
+        ).to(self.device)
+        self.model.load_state_dict(torch.load(self.checkpoint_path, map_location=self.device))
+        self.model.eval()
+        return self
+
+    @torch.no_grad()
+    def predict_proba_single(self, prompt: str, options: List[str]) -> np.ndarray:
+        """Returns a (5,) joint-softmax probability array in A-E order."""
+        if self.model is None:
+            self.load()
+
+        prompts_x5, options_x5 = build_roberta_inputs_single(prompt, options)
+        enc = self.tokenizer(
+            prompts_x5, options_x5,
+            max_length=ROBERTA_CFG["max_len"],
+            truncation=True,
+            padding="max_length",
+            return_tensors="pt",
+        )
+        # Reshape to (batch=1, num_choices=5, seq_len) as AutoModelForMultipleChoice expects
+        input_ids = enc["input_ids"].unsqueeze(0).to(self.device)
+        attention_mask = enc["attention_mask"].unsqueeze(0).to(self.device)
+
+        out = self.model(input_ids=input_ids, attention_mask=attention_mask)
+        proba = torch.softmax(out.logits, dim=1).cpu().numpy()[0]
+        return proba
+
+    def predict_top3_single(self, prompt: str, options: List[str]) -> List[str]:
+        from ..config import REVERSE_MAP
+        proba = self.predict_proba_single(prompt, options)
+        top3_idx = np.argsort(proba)[-3:][::-1]
+        return [REVERSE_MAP[i] for i in top3_idx]

src/models/tfidf.py

@@ -0,0 +1,51 @@
+"""
+models/tfidf.py
+================
+TF-IDF + Logistic Regression baseline wrapper (02_baseline.ipynb).
+Loads the pickled vectorizer + classifier and exposes a predict_proba()
+that returns per-option probabilities in A-E order.
+"""
+
+from pathlib import Path
+from typing import List
+
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.linear_model import LogisticRegression
+
+from ..config import TFIDF_CFG
+from ..preprocessing import build_tfidf_text_single
+from ..utils import load_pickle
+
+
+class TFIDFModel:
+    """Wraps the fitted TfidfVectorizer + LogisticRegression pair."""
+
+    def __init__(self, model_path: Path = None, vectorizer_path: Path = None,
+                 strategy: str = None):
+        self.model_path = Path(model_path or TFIDF_CFG["model_path"])
+        self.vectorizer_path = Path(vectorizer_path or TFIDF_CFG["vectorizer_path"])
+        self.strategy = strategy or TFIDF_CFG["text_strategy"]
+        self.model: LogisticRegression = None
+        self.vectorizer: TfidfVectorizer = None
+
+    def load(self) -> "TFIDFModel":
+        self.model = load_pickle(self.model_path)
+        self.vectorizer = load_pickle(self.vectorizer_path)
+        return self
+
+    def predict_proba_single(self, prompt: str, options: List[str]) -> np.ndarray:
+        """Returns a (5,) probability array in A-E order for one question."""
+        if self.model is None or self.vectorizer is None:
+            self.load()
+        text = build_tfidf_text_single(prompt, options, strategy=self.strategy)
+        X = self.vectorizer.transform([text])
+        proba = self.model.predict_proba(X)[0]
+        return proba
+
+    def predict_top3_single(self, prompt: str, options: List[str]) -> List[str]:
+        """Returns top-3 option letters (e.g. ['B', 'D', 'A']) for one question."""
+        from ..config import REVERSE_MAP
+        proba = self.predict_proba_single(prompt, options)
+        top3_idx = np.argsort(proba)[-3:][::-1]
+        return [REVERSE_MAP[i] for i in top3_idx]

src/predict.py

@@ -0,0 +1,46 @@
+"""
+predict.py
+==========
+The single public entrypoint used by app.py. Everything else in src/ is
+plumbing — this is the one function a Streamlit UI (or any other frontend)
+needs to call.
+"""
+
+from typing import List, Dict
+
+from .inference import predict_all_models
+from .ensemble import rank_ensemble_with_scores
+from .config import LEADERBOARD_SCORES
+
+
+def predict(prompt: str, options: List[str]) -> Dict:
+    """
+    Full pipeline: run all 4 models -> ensemble -> return everything the UI
+    needs to render (final answer, per-model breakdown, confidence scores).
+
+    Args:
+        prompt:  the question text
+        options: list of 5 option strings, in A, B, C, D, E order
+
+    Returns:
+        {
+            "final_top3": ["B", "A", "D"],
+            "per_model_top3": {
+                "tfidf": [...], "lstm": [...], "deberta": [...], "roberta": [...]
+            },
+            "ensemble_scores": {"B": 0.62, "A": 0.41, ...},
+            "leaderboard_scores": {...}   # for display / transparency
+        }
+    """
+    if len(options) != 5:
+        raise ValueError(f"Expected exactly 5 options (A-E), got {len(options)}")
+
+    per_model_top3 = predict_all_models(prompt, options)
+    final_top3, ensemble_scores = rank_ensemble_with_scores(per_model_top3)
+
+    return {
+        "final_top3": final_top3,
+        "per_model_top3": per_model_top3,
+        "ensemble_scores": ensemble_scores,
+        "leaderboard_scores": LEADERBOARD_SCORES,
+    }

src/preprocessing.py

@@ -0,0 +1,163 @@
+"""
+preprocessing.py
+================
+Text cleaning and prompt-building functions, extracted from the notebooks.
+Every model needs a slightly different input format:
+
+  - TF-IDF     : one flat string per question (3 strategies tried)
+  - LSTM       : prompt + all 5 options concatenated, word-tokenized
+  - DeBERTa    : question expanded into 5 (prompt, single option) pairs
+  - RoBERTa    : 5 stacked (prompt, option) pairs per question (multiple-choice)
+"""
+
+from typing import List
+import pandas as pd
+
+from .config import OPTION_COLS, ANSWER_MAP
+
+
+# ---------------------------------------------------------------------------
+# Shared cleaning (identical in all 4 notebooks)
+# ---------------------------------------------------------------------------
+def clean_text(text: str) -> str:
+    """Lowercase + strip whitespace. Applied to every text column at load time."""
+    return str(text).lower().strip()
+
+
+def clean_dataframe(df: pd.DataFrame, cols: List[str] = None) -> pd.DataFrame:
+    """Apply clean_text() to prompt + option columns of a dataframe (returns a copy)."""
+    cols = cols or (["prompt"] + OPTION_COLS)
+    df = df.copy()
+    for col in cols:
+        if col in df.columns:
+            df[col] = df[col].astype(str).str.lower().str.strip()
+    return df
+
+
+def stratified_split(train_df: pd.DataFrame, val_size: float = 0.2, seed: int = 42):
+    """
+    Manual per-class stratified split, identical across all 4 training notebooks.
+    Keeps the per-answer-letter distribution the same in train and val.
+    """
+    import numpy as np
+
+    np.random.seed(seed)
+    train_idx, val_idx = [], []
+    for ans in "ABCDE":
+        idx = train_df[train_df["answer"] == ans].index.tolist()
+        np.random.shuffle(idx)
+        cut = int(len(idx) * (1.0 - val_size))
+        train_idx += idx[:cut]
+        val_idx += idx[cut:]
+
+    tr = train_df.loc[train_idx].reset_index(drop=True)
+    va = train_df.loc[val_idx].reset_index(drop=True)
+    return tr, va
+
+
+# ---------------------------------------------------------------------------
+# TF-IDF text-builder strategies (02_baseline.ipynb)
+# ---------------------------------------------------------------------------
+def build_v1_simple(df: pd.DataFrame):
+    """prompt + all options concatenated (naive baseline)"""
+    return (df["prompt"] + " " + df["A"] + " " + df["B"] + " " +
+            df["C"] + " " + df["D"] + " " + df["E"]).values
+
+
+def build_v2_repeated(df: pd.DataFrame):
+    """prompt repeated before EACH option: 'Q optA Q optB Q optC Q optD Q optE'"""
+    q = df["prompt"]
+    return (q + " " + df["A"] + " " + q + " " + df["B"] + " " +
+            q + " " + df["C"] + " " + q + " " + df["D"] + " " + q + " " + df["E"]).values
+
+
+def build_v3_labeled(df: pd.DataFrame):
+    """explicit 'option a: ...' labels + trigrams — best-performing strategy"""
+    return (df["prompt"] +
+            " option a: " + df["A"] +
+            " option b: " + df["B"] +
+            " option c: " + df["C"] +
+            " option d: " + df["D"] +
+            " option e: " + df["E"]).values
+
+
+TEXT_BUILDERS = {
+    "v1_simple": build_v1_simple,
+    "v2_repeated": build_v2_repeated,
+    "v3_labeled": build_v3_labeled,
+}
+
+
+def build_tfidf_text(df: pd.DataFrame, strategy: str = "v3_labeled"):
+    """Dispatch to the chosen TF-IDF text-builder strategy."""
+    return TEXT_BUILDERS[strategy](df)
+
+
+def build_tfidf_text_single(prompt: str, options: List[str], strategy: str = "v3_labeled") -> str:
+    """Same as build_tfidf_text but for a single question (used at inference time)."""
+    a, b, c, d, e = options
+    prompt = clean_text(prompt)
+    a, b, c, d, e = [clean_text(x) for x in (a, b, c, d, e)]
+
+    if strategy == "v1_simple":
+        return f"{prompt} {a} {b} {c} {d} {e}"
+    if strategy == "v2_repeated":
+        return (f"{prompt} {a} {prompt} {b} {prompt} {c} "
+                f"{prompt} {d} {prompt} {e}")
+    if strategy == "v3_labeled":
+        return (f"{prompt} option a: {a} option b: {b} option c: {c} "
+                f"option d: {d} option e: {e}")
+    raise ValueError(f"Unknown strategy: {strategy}")
+
+
+# ---------------------------------------------------------------------------
+# LSTM: prompt + all options concatenated (03_lstm.ipynb)
+# ---------------------------------------------------------------------------
+def build_lstm_text(prompt: str, options: List[str]) -> str:
+    """Combined text used by the LSTM tokenizer: prompt + A + B + C + D + E."""
+    prompt = clean_text(prompt)
+    a, b, c, d, e = [clean_text(x) for x in options]
+    return f"{prompt} {a} {b} {c} {d} {e}"
+
+
+# ---------------------------------------------------------------------------
+# DeBERTa: expand each question into 5 (prompt, option) pairs (04_DeBERTa.ipynb)
+# ---------------------------------------------------------------------------
+def expand_pairs(df: pd.DataFrame, is_test: bool = False) -> pd.DataFrame:
+    """
+    Convert each question row into 5 rows: one per option.
+    text  = 'prompt option_text'
+    label = 1 if this option is correct else 0 (only when not is_test)
+    """
+    rows = []
+    for _, r in df.iterrows():
+        correct = r.get("answer", None)
+        for opt in OPTION_COLS:
+            rows.append({
+                "id": r["id"],
+                "option": opt,
+                "text": f"{r['prompt']} {r[opt]}",
+                "label": int(opt == correct) if not is_test else -1,
+            })
+    return pd.DataFrame(rows)
+
+
+def build_deberta_pairs_single(prompt: str, options: List[str]) -> List[str]:
+    """Same expansion, but for a single question at inference time.
+    Returns a list of 5 strings, one per option, in A-E order."""
+    prompt = clean_text(prompt)
+    return [f"{prompt} {clean_text(opt)}" for opt in options]
+
+
+# ---------------------------------------------------------------------------
+# RoBERTa: 5 stacked (prompt, option) pairs for AutoModelForMultipleChoice
+# (05_RoBERTa.ipynb) — tokenizer handles the pairing, this just prepares lists
+# ---------------------------------------------------------------------------
+def build_roberta_inputs_single(prompt: str, options: List[str]):
+    """
+    Returns (prompts_x5, options_x5) ready to feed straight into the
+    tokenizer as tokenizer([prompt]*5, options, ...).
+    """
+    prompt = clean_text(prompt)
+    options = [clean_text(opt) for opt in options]
+    return [prompt] * 5, options

src/utils.py

@@ -0,0 +1,83 @@
+"""
+utils.py
+========
+Shared helpers used across every model module: reproducibility seeding,
+device detection, checkpoint save/load, and a simple logger.
+"""
+
+import logging
+import os
+import random
+import pickle
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+try:
+    import torch
+except ImportError:
+    torch = None
+
+
+def seed_everything(seed: int = 42) -> None:
+    """Set every RNG (python, numpy, torch) for reproducibility.
+    Matches the set_seed()/torch.manual_seed() calls in all training notebooks.
+    """
+    random.seed(seed)
+    np.random.seed(seed)
+    os.environ["PYTHONHASHSEED"] = str(seed)
+
+    if torch is not None:
+        torch.manual_seed(seed)
+        if torch.cuda.is_available():
+            torch.cuda.manual_seed_all(seed)
+        torch.backends.cudnn.deterministic = True
+        torch.backends.cudnn.benchmark = False
+
+
+def get_device():
+    """Return 'cuda' if available else 'cpu' (same logic as every notebook)."""
+    if torch is None:
+        return "cpu"
+    return torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def save_model(model, path: Path) -> None:
+    """Save a torch model's state_dict to disk (creates parent dirs)."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    torch.save(model.state_dict(), path)
+
+
+def load_model(model, path: Path, device=None):
+    """Load a torch model's state_dict from disk in-place and return it in eval mode."""
+    device = device or get_device()
+    model.load_state_dict(torch.load(path, map_location=device))
+    model.to(device)
+    model.eval()
+    return model
+
+
+def save_pickle(obj: Any, path: Path) -> None:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "wb") as f:
+        pickle.dump(obj, f)
+
+
+def load_pickle(path: Path) -> Any:
+    with open(path, "rb") as f:
+        return pickle.load(f)
+
+
+def get_logger(name: str = "smart_mcq_solver") -> logging.Logger:
+    """Simple stdout logger, reused everywhere instead of print()."""
+    logger = logging.getLogger(name)
+    if not logger.handlers:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter("[%(asctime)s] %(levelname)s - %(message)s", "%H:%M:%S")
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+        logger.setLevel(logging.INFO)
+    return logger