Update app.py

app.py

@@ -4,6 +4,9 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
 from datasets import load_dataset
 import torch
 import re
+import json
+import pandas as pd
+import matplotlib.pyplot as plt
 
 # Cache to avoid reloading the model
 model_cache = {}
@@ -20,8 +23,7 @@ def load_model(model_id):
     return generator
 
 def format_prompt(item):
-    system_instruction = "
-Only answer with a single letter: A, B, C, or D."
+    system_instruction = " Only answer with a single letter: A, B, C, or D."
     prompt = f"{item['question']}
 A. {item['choices'][0]}
 B. {item['choices'][1]}
@@ -55,10 +57,23 @@ def evaluate(model_id, sample_count, config_name):
 
 def run(model_id, sample_count, config_name):
     score, details = evaluate(model_id, sample_count, config_name)
-    formatted = "\n\n".join([
-        f"### Question:\n{q}\n\n**Model Answer:** {o}\n**Expected:** {a}\n**Predicted:** {g}\n**Correct:** {c}"
+    formatted = "
+
+".join([
+        f"### Question:
+{q}
+
+**Model Answer:** {o}
+**Expected:** {a}
+**Predicted:** {g}
+**Correct:** {c}"
         for q, o, a, g, c in details
     ])
+    accuracy_value = float(score.split()[1][:-1])
+    record = {"model_id": model_id, "subject": config_name, "accuracy": accuracy_value}
+    with open("eval.jsonl", "a") as f:
+        f.write(json.dumps(record) + "
+")
     return score, formatted
 
 def save_text(text):
@@ -105,4 +120,23 @@ with gr.Blocks(css="body {font-family: Inter, sans-serif; padding: 1em; max-widt
     run_button.click(run, inputs=[model_id, sample_count, config_name], outputs=[acc_output, detail_output])
     download_button.click(save_text, inputs=detail_output, outputs=gr.File())
 
+    with gr.Row():
+        leaderboard_plot = gr.Plot(label="Leaderboard Chart")
+        leaderboard_table = gr.Dataframe(headers=["Model ID", "Subject", "Accuracy"], interactive=False)
+
+    def load_leaderboard():
+        try:
+            df = pd.read_json("eval.jsonl", lines=True)
+            df_sorted = df.sort_values(by="accuracy", ascending=False).head(10)
+            fig, ax = plt.subplots()
+            ax.barh(df_sorted['model_id'], df_sorted['accuracy'])
+            ax.set_xlabel("Accuracy")
+            ax.set_ylabel("Model")
+            ax.set_title("Top 10 Models")
+            return fig, df_sorted
+        except Exception as e:
+            return plt.figure(), pd.DataFrame(columns=["model_id", "subject", "accuracy"])
+
+    demo.load(load_leaderboard, inputs=[], outputs=[leaderboard_plot, leaderboard_table])
+
 demo.launch()