Update app.py

app.py

@@ -1,38 +1,48 @@
 import gradio as gr
 from transformers import pipeline
 from datasets import load_dataset
+import torch
 
-# ✅ Load the smaller model (fits in 16GB)
-model_name = "Qwen/Qwen2.5-3B"
-pipe = pipeline("text-generation", model=model_name, device=0)
-
-# ✅ Load the GTA dataset (correct split is 'train')
+# Load GTA dataset
 gta = load_dataset("Jize1/GTA", split="train")
 
-# ✅ Inference function
-def run_model(input_text, use_gta_idx):
-    if use_gta_idx:
-        try:
-            idx = int(input_text)
-            question = gta[idx]["dialogs"][0]["content"]
-        except Exception as e:
-            return f"❌ Invalid index (0–{len(gta)-1}): {e}"
-    else:
-        question = input_text.strip()
-
-    output = pipe(question, max_new_tokens=256, do_sample=True)
-    return f"**Question:** {question}\n\n**Response:**\n{output[0]['generated_text']}"
-
-# ✅ Gradio UI
+def evaluate_model(model_name, num_samples):
+    try:
+        pipe = pipeline("text-generation", model=model_name, device=0 if torch.cuda.is_available() else -1)
+
+        correct = 0
+        total = 0
+        log = []
+
+        for i in range(min(num_samples, len(gta))):
+            query = gta[i]["dialogs"][0]["content"]
+            gt_answers = gta[i]["gt_answer"].get("whitelist", [])
+            flat_gt = {ans.strip().lower() for group in gt_answers for ans in group if isinstance(ans, str)}
+
+            # Generate model output
+            out = pipe(query, max_new_tokens=128, do_sample=False)[0]["generated_text"].strip().lower()
+
+            # Match: exact substring match with any whitelist answer
+            matched = any(gt in out for gt in flat_gt)
+
+            log.append(f"### Query {i}\n**Input**: {query}\n**Prediction**: {out}\n**GT**: {flat_gt}\n**✔️ Correct**: {matched}\n")
+            correct += int(matched)
+            total += 1
+
+        acc = round((correct / total) * 100, 2)
+        summary = f"### 🔍 GTA Answer Accuracy (AnsAcc) for `{model_name}`: **{acc}%** on {total} queries\n\n---\n"
+        return summary + "\n".join(log)
+
+    except Exception as e:
+        return f"❌ Evaluation failed: {e}"
+
 with gr.Blocks() as demo:
-    gr.Markdown("# 🤖 GTA Reasoning Demo (Qwen2.5‑3B + GTA Dataset)")
-    gr.Markdown("Enter a custom question or choose a sample from the GTA dataset (index 0–228).")
-    with gr.Row():
-        input_text = gr.Textbox(label="Your input or GTA index")
-        use_index = gr.Checkbox(label="Treat input as GTA index", value=False)
-    run_btn = gr.Button("Generate")
+    gr.Markdown("# 🧪 Real GTA Evaluation (Answer Accuracy Only)")
+    model_input = gr.Textbox(label="Enter Hugging Face Model Name", value="Qwen/Qwen2.5-3B")
+    sample_count = gr.Slider(label="Number of GTA samples to evaluate", minimum=1, maximum=229, value=10, step=1)
     output_md = gr.Markdown()
-    
-    run_btn.click(fn=run_model, inputs=[input_text, use_index], outputs=[output_md])
+
+    model_input.change(fn=evaluate_model, inputs=[model_input, sample_count], outputs=output_md)
+    sample_count.change(fn=evaluate_model, inputs=[model_input, sample_count], outputs=output_md)
 
 demo.launch()