add the Open LLM bench

app.py

@@ -5,6 +5,7 @@ import numpy as np
 import pandas as pd
 import gradio as gr
 import pandas as pd 
+import json 
 from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS  
 
 LAST_UPDATED = "Feb 28th 2024"
@@ -66,27 +67,35 @@ def make_clickable_model(model_name, model_info):
         return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_info[model_name]["pretty_name"]}</a>'
 
 
-def build_demo(original_df, TYPES):
+def build_demo(original_df, full_df, TYPES):
     with gr.Blocks(theme=gr.themes.Soft(), css=css) as demo:
         # gr.HTML(BANNER, elem_id="banner")
         gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
-        # with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        #     with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0):
-        leaderboard_table = gr.components.Dataframe(
-            value=original_df,
-            datatype=TYPES,
-            height=1000,
-            wrap=False,
-            elem_id="leaderboard-table",
-            interactive=False,
-            visible=True,
-            min_width=60,
-            )
-
-            # with gr.TabItem("📈 Metrics", elem_id="od-benchmark-tab-table", id=1):
-            #     gr.Markdown(METRICS_TAB_TEXT, elem_classes="markdown-text")
- 
+        with gr.Tabs(elem_classes="tab-buttons") as tabs:
+            with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0):
+                leaderboard_table = gr.components.Dataframe(
+                    value=original_df,
+                    datatype=TYPES,
+                    height=1000,
+                    wrap=False,
+                    elem_id="leaderboard-table",
+                    interactive=False,
+                    visible=True,
+                    min_width=60,
+                    )
+
+            with gr.TabItem("🐑 URIAL + 🤗 OpenLLM", elem_id="od-benchmark-tab-table", id=1):
+                leaderboard_table_full = gr.components.Dataframe(
+                    value=full_df,
+                    datatype=TYPES,
+                    height=1000,
+                    wrap=False,
+                    elem_id="leaderboard-table-full",
+                    interactive=False,
+                    visible=True,
+                    min_width=60,
+                    ) 
 
         gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text-small")
         
@@ -108,9 +117,34 @@ if __name__ == "__main__":
     parser.add_argument("--result_file", help="Path to results table", default="leaderboard_data.jsonl")
     args = parser.parse_args()
 
-    bench_results = args.result_file
+    all_model_hf_ids = {v["hf_name"]: k  for k, v in model_info.items()}
+
+    # Load Open LLM Leaderboard 
+    with open("open-llm-leaderboard.json") as f:
+        open_llm_leaderbaord = json.load(f)
+    full_leaderboard = {}
+    for item in open_llm_leaderbaord:
+        if item["Model"] in all_model_hf_ids:
+            # print(item["Model"])
+            # print(item["Average \u2b06\ufe0f"])
+            full_bench_item = {}
+            # full_bench_item["hf_name"] = item["Model"]
+            full_bench_item["model_name"] = all_model_hf_ids[item["Model"]]
+            tasks = ["HellaSwag", "ARC", "Winogrande", "TruthfulQA", "MMLU", "GSM8K"]
+            for task in tasks:
+                full_bench_item[task] = item[task]
+            full_bench_item["HF_AVG"] = item["Average \u2b06\ufe0f"]
+            full_leaderboard[all_model_hf_ids[item["Model"]]] = full_bench_item
+    # Load URIAL Leaderboard
+    with open("leaderboard_data.jsonl") as f:
+        for line in f:
+            item = json.loads(line)  
+            if item["model"] in full_leaderboard:
+                full_leaderboard[item["model"]]["URIAL_AVG"] = item["Overall"]   
  
-    original_df = pd.read_json(bench_results, lines=True)
+
+    # Process the URIAL Benchmark Tab 
+    original_df = pd.read_json(args.result_file, lines=True)
 
     print(original_df.columns)
 
@@ -119,21 +153,29 @@ if __name__ == "__main__":
             original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x, model_info)))
         else: 
             original_df[col] = original_df[col].apply(formatter) # For numerical values
-    
     # Define the first column explicitly, add 'Overall' as the second column, and then append the rest excluding 'Overall'
     new_order = [original_df.columns[0], 'Overall'] + [col for col in original_df.columns if col not in [original_df.columns[0], 'Overall']]
-
     # Reorder the DataFrame columns using the new order
     reordered_df = original_df[new_order]
-
     reordered_df.sort_values(by='Overall', inplace=True, ascending=False)
-
     reordered_df.rename(columns=column_names, inplace=True)
 
+    # Process the Full Benchmark Tab
+    full_df = pd.DataFrame(full_leaderboard).T
+    full_df = full_df.reset_index()
+    full_df.rename(columns={"index": "model"}, inplace=True)
+    full_df = full_df[["model",  "URIAL_AVG", "HF_AVG", "HellaSwag", "ARC", "Winogrande", "TruthfulQA", "MMLU", "GSM8K"]]
+    full_df.sort_values(by='URIAL_AVG', inplace=True, ascending=False)
+    full_df["model"] = full_df["model"].apply(lambda x: make_clickable_model(x, model_info))
+    full_df.rename(columns=column_names, inplace=True)
+    # apply formatter to numerical columns
+    for col in full_df.columns:
+        if col not in ["Model"]:
+            full_df[col] = full_df[col].apply(formatter) # For numerical values
     # COLS = [c.name for c in fields(AutoEvalColumn)]
     # TYPES = [c.type for c in fields(AutoEvalColumn)]
 
     TYPES = ["markdown", "number"]
-    demo = build_demo(reordered_df, TYPES)
+    demo = build_demo(reordered_df, full_df, TYPES)
     demo.launch(share=args.share)
  

scrape-open-llm-leaderboard

@@ -0,0 +1 @@
+Subproject commit ed2132d6989c336d684c3e3c4681aa458ace24ca