Upload 2 files

app.py

@@ -0,0 +1,69 @@
+import gradio as gr
+import pandas as pd
+
+
+def load_and_process_data(file_path):
+    # Load the leaderboard data
+    df = pd.read_pickle(file_path)
+
+    # Group by 'lmm' and 'question' to calculate mean accuracy
+    accuracy_df = (
+        df.groupby(["lmm", "question"])["accepted_by_judge"].mean().reset_index()
+    )
+    accuracy_df = accuracy_df.rename(columns={"accepted_by_judge": "accuracy"})
+    accuracy_df["accuracy"] = (accuracy_df["accuracy"] * 100).round(1)
+
+    # Group by 'lmm' to calculate the count of images
+    image_count_df = df.groupby("lmm")["image"].nunique().reset_index()
+    image_count_df = image_count_df.rename(columns={"image": "Total Images"})
+
+    return accuracy_df, image_count_df
+
+
+def expand_and_format_df(accuracy_df, image_count_df):
+    # Pivot and format the accuracy dataframe
+    expanded_df = accuracy_df.pivot(index="lmm", columns="question", values="accuracy")
+    expanded_df["Average"] = expanded_df.mean(axis=1).round(1)
+    expanded_df = expanded_df.sort_values(by="Average", ascending=False).reset_index()
+    expanded_df.columns.name = None
+
+    # Merge the 'total_images' column
+    final_df = pd.merge(expanded_df, image_count_df, on="lmm")
+
+    return final_df.rename(columns={"lmm": "Model"})
+
+
+def map_model_names(df, name_dict):
+    # Map model names using the provided dictionary
+    df["Model"] = df["Model"].map(name_dict)
+    return df
+
+
+# Dictionary for renaming models
+name_dict = {
+    "gpt4v": "GPT-4V(ision)",
+    "llava": "LLaVA-1.5-13B",
+    "llava-7b": "LLaVA-1.5-7B",
+    "Long-SPHINX": "Long-SPHINX",
+    "SPHINX": "SPHINX",
+    "OtterHD": "OtterHD",
+    "minigpt4v2": "MiniGPT4v2",
+    "InstructBLIP-13B": "InstructBLIP-13B",
+    "InstructBLIP": "InstructBLIP-7B",
+    "qwen": "Qwen-VL-Chat",
+    "fuyu-8b": "Fuyu-8B",
+}
+
+# Processing steps
+accuracy_df, image_count_df = load_and_process_data("raw_outputs.pkl")
+final_df = expand_and_format_df(accuracy_df, image_count_df)
+final_df = map_model_names(final_df, name_dict)
+
+
+# Gradio interface
+with gr.Blocks() as demo:
+    gr.Markdown("# GlitchBench Leaderboard")
+    with gr.Row():
+        gr.Dataframe(final_df)
+
+demo.launch()

raw_outputs.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1777b3c9404d0d8ebbe286fd42114767e70f19c428af736c64273bc414af25e
+size 22207169