app.py

"""Smart MCQ Solver - Gradio demo.

Loads the fine-tuned DeBERTa-v3-large multiple-choice model from the Hub and
ranks five candidate answers for a question.
"""
import os
import numpy as np
import torch
import gradio as gr
from transformers import AutoTokenizer, AutoModelForMultipleChoice

# Available only on ZeroGPU hardware. On CPU-basic the import fails and we stay on CPU,
# so this one file runs unchanged on either.
try:
    import spaces
    HAS_ZEROGPU = True
except ImportError:
    HAS_ZEROGPU = False

MODEL_ID = os.environ.get("MODEL_ID", "SriragData/smart-mcq-deberta-v3-large")
MAX_LEN = 256
LAB = list("ABCDE")

torch.set_num_threads(max(1, (os.cpu_count() or 2) // 2))

print(f"loading {MODEL_ID} ...")
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

# fp32 is forced explicitly on purpose: transformers honours the dtype recorded in
# config.json, and an fp16 DeBERTa-v3 collapses to a constant output with no error
# raised. The kwarg was renamed (torch_dtype -> dtype) between versions, so try both
# rather than pinning ourselves to one.
try:
    model = AutoModelForMultipleChoice.from_pretrained(MODEL_ID, dtype=torch.float32)
except TypeError:
    model = AutoModelForMultipleChoice.from_pretrained(MODEL_ID, torch_dtype=torch.float32)
model = model.eval()
assert next(model.parameters()).dtype == torch.float32, "model did not load in fp32"
print("loaded. params:", sum(p.numel() for p in model.parameters()))


TEMPLATES = [
    "Pick the best possible answer:", "Select the most accurate option:",
    "Identify the correct statement:", "Determine the correct option:",
    "Choose the correct answer:", "Which of the following is correct?",
]


def clean(prompt: str) -> str:
    """Strip the boilerplate wrappers the training data was generated with."""
    for t in TEMPLATES:
        prompt = prompt.replace(t, "")
    return prompt.strip()


@torch.no_grad()
def solve(question, a, b, c, d, e):
    options = [a, b, c, d, e]
    if not question or not question.strip():
        return "Enter a question.", None
    if any(not str(o).strip() for o in options):
        return "All five options are required.", None

    # On ZeroGPU, CUDA only exists inside the decorated call, so resolve the device here
    # rather than at import time.
    device = "cuda" if torch.cuda.is_available() else "cpu"
    m = model.to(device)

    q = clean(question)
    enc = tokenizer([q] * 5, [str(o) for o in options],
                    truncation=True, max_length=MAX_LEN,
                    padding=True, return_tensors="pt")
    logits = m(input_ids=enc["input_ids"].unsqueeze(0).to(device),
               attention_mask=enc["attention_mask"].unsqueeze(0).to(device)).logits[0]
    probs = torch.softmax(logits.float().cpu(), dim=-1).numpy()

    order = np.argsort(-probs)
    top3 = " ".join(LAB[i] for i in order[:3])
    verdict = (f"### {LAB[order[0]]} — {options[order[0]]}\n\n"
               f"**MAP@3 submission format:** `{top3}`")

    rows = [[LAB[i], str(options[i])[:200], round(float(probs[i]), 4),
             int(np.where(order == i)[0][0]) + 1] for i in range(5)]
    rows.sort(key=lambda r: r[3])
    return verdict, rows


if HAS_ZEROGPU:
    # Requests a GPU slice for the duration of the call. Applied programmatically so the
    # same file still imports on CPU-basic hardware, where `spaces` does not exist.
    solve = spaces.GPU(duration=60)(solve)


EXAMPLES = [
    ["Which philosopher argued that existence precedes essence?",
     "Immanuel Kant", "Jean-Paul Sartre", "David Hume", "Rene Descartes", "Baruch Spinoza"],
    ["What is the primary function of mitochondria in a eukaryotic cell?",
     "Protein synthesis", "Storage of genetic material",
     "Production of ATP through oxidative phosphorylation",
     "Breakdown of cellular waste", "Regulation of cell division"],
    ["According to the second law of thermodynamics, what happens to the entropy "
     "of an isolated system over time?",
     "It decreases to zero", "It remains exactly constant",
     "It never decreases", "It oscillates periodically", "It becomes negative"],
]

CSS = """
.gradio-container {max-width: 1000px !important}
footer {visibility: hidden}
"""

with gr.Blocks(title="Smart MCQ Solver", css=CSS, theme=gr.themes.Soft()) as demo:
    gr.Markdown(
        "# Smart MCQ Solver\n"
        "Fine-tuned **DeBERTa-v3-large** ranking five candidate answers. "
        "Scored by MAP@3: credit 1, 1/2, 1/3 if the correct option is ranked 1st, 2nd or 3rd.\n\n"
        "*3-fold grouped CV MAP@3 **0.7567** — random baseline is 0.3667. "
        "Closed-book: it answers from its weights, with no retrieval and no abstention.*"
    )
    with gr.Row():
        with gr.Column(scale=3):
            question = gr.Textbox(label="Question", lines=2,
                                  placeholder="Ask a science or philosophy question...")
            with gr.Row():
                a = gr.Textbox(label="A")
                b = gr.Textbox(label="B")
            with gr.Row():
                c = gr.Textbox(label="C")
                d = gr.Textbox(label="D")
            e = gr.Textbox(label="E")
            btn = gr.Button("Rank the options", variant="primary")
        with gr.Column(scale=2):
            verdict = gr.Markdown(label="Answer")
            table = gr.Dataframe(headers=["Option", "Text", "Probability", "Rank"],
                                 datatype=["str", "str", "number", "number"],
                                 label="All five, ranked", wrap=True)

    btn.click(solve, [question, a, b, c, d, e], [verdict, table])
    gr.Examples(EXAMPLES, inputs=[question, a, b, c, d, e])

    gr.Markdown(
        "---\n"
        "**Limitations.** Trained on 2,000 rows covering 252 unique questions, so coverage "
        "is narrow. It has no way to say *I don't know* — it will rank five options "
        "confidently even when it knows nothing about the topic. Reported scores reflect a "
        "dataset whose test split overlaps its training split heavily; the leakage-free "
        "estimate for this project is 0.6817."
    )

if __name__ == "__main__":
    demo.launch()