update leaderboard

app.py

@@ -168,7 +168,7 @@ with demo:
                         with gr.Column(min_width=320):
                             shown_phenotypes = gr.CheckboxGroup(
                                 choices=sorted(set([
-                                    c.task.value.phenotype_name
+                                    c.task.value.phenotype
                                     for c in fields(AutoEvalColumn)
                                     if not c.hidden and not c.never_hidden and c.is_task
                                 ])),
@@ -178,12 +178,12 @@ with demo:
                             )
                             shown_metrics = gr.CheckboxGroup(
                                 choices=sorted(set([
-                                    c.task.value.metric_name
+                                    c.task.value.metric.upper()
                                     for c in fields(AutoEvalColumn)
                                     if not c.hidden and not c.never_hidden and c.is_task
                                 ])),
                                 value=sorted(set([
-                                    c.task.value.metric_name
+                                    c.task.value.metric.upper()
                                     for c in fields(AutoEvalColumn)
                                     if not c.hidden and not c.never_hidden and c.is_task
                                 ])),
@@ -212,42 +212,41 @@ with demo:
                             value=True, label="Show gated/private/deleted models", interactive=True
                         )
                 with gr.Column(min_width=320):
-                    with gr.Column(min_width=320):
-                        filter_features = gr.CheckboxGroup(
-                            label="Features Set",
-                            choices=[("Baseline (age, sex, BMI)", "baseline"), ("Expanded (age, sex, BMI, HDL, LDL, total-cholesterol, triglycerides, diastolic-blood-pressure, smoking-status, snoring, insomnia, daytime-napping, sleep-duration, chronotype)", "expanded")],
-                            value=["baseline"],
-                            interactive=True,
-                            elem_id="filter-feature-set",
-                        )
-                        filter_nb_shots = gr.CheckboxGroup(
-                            label="Number of shots",
-                            choices=[("Zero-shot", 0), ("10-shot", 10), ("All", -1)],
-                            value=[0],
-                            interactive=True,
-                            elem_id="filter-nb-shots",
-                        )
-                        filter_columns_type = gr.CheckboxGroup(
-                            label="Model types",
-                            choices=[t.to_str() for t in ModelType],
-                            value=[t.to_str() for t in ModelType],
-                            interactive=True,
-                            elem_id="filter-columns-type",
-                        )
-                        filter_columns_precision = gr.CheckboxGroup(
-                            label="Precision",
-                            choices=[i.value.name for i in Precision],
-                            value=[i.value.name for i in Precision],
-                            interactive=True,
-                            elem_id="filter-columns-precision",
-                        )
-                        filter_columns_size = gr.CheckboxGroup(
-                            label="Model sizes (in billions of parameters)",
-                            choices=list(NUMERIC_INTERVALS.keys()),
-                            value=list(NUMERIC_INTERVALS.keys()),
-                            interactive=True,
-                            elem_id="filter-columns-size",
-                        )
+                    filter_features = gr.CheckboxGroup(
+                        label="Features Set",
+                        choices=[("Baseline (Age, Sex, BMI)", "baseline"), ("Expanded (Age, Sex, BMI, HDL, LDL, Total cholesterol, Triglycerides, Diastolic blood pressure, Smoking status, Snoring, Insomnia, Daytime napping, Sleep duration, Chronotype)", "expanded")],
+                        value=["baseline", "expanded"],
+                        interactive=True,
+                        elem_id="filter-feature-set",
+                    )
+                    filter_nb_shots = gr.CheckboxGroup(
+                        label="Number of shots",
+                        choices=[("Zero-Shot", 0), ("10-Shot", 10), ("All", -1)],
+                        value=[0],
+                        interactive=True,
+                        elem_id="filter-nb-shots",
+                    )
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Model types",
+                        choices=[t.to_str() for t in ModelType],
+                        value=[t.to_str() for t in ModelType],
+                        interactive=True,
+                        elem_id="filter-columns-type",
+                    )
+                    filter_columns_precision = gr.CheckboxGroup(
+                        label="Precision",
+                        choices=[i.value.name for i in Precision],
+                        value=[i.value.name for i in Precision],
+                        interactive=True,
+                        elem_id="filter-columns-precision",
+                    )
+                    filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
@@ -260,12 +259,6 @@ with demo:
                 interactive=False,
                 visible=True,
             )
-            
-            # Plotting the curves
-            # gr.Plot(
-            #     plot_curves(),
-            #     elem_id="plot-curves"
-            # )
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
@@ -309,6 +302,29 @@ with demo:
                     queue=True,
                 )
 
+        # with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=2):
+        #     with gr.Row():
+        #         gr.Plot(
+        #             plot_curves(),
+        #             elem_id="plot-curves"
+        #         )
+                # with gr.Column():
+                #     plot_df = load_and_create_plots()
+                #     chart = create_metric_plot_obj(
+                #         plot_df,
+                #         [AutoEvalColumn.average.name],
+                #         title="Average of Top Scores and Human Baseline Over Time (from last update)",
+                #     )
+                #     gr.Plot(value=chart, min_width=500)
+                # with gr.Column():
+                #     plot_df = load_and_create_plots()
+                #     chart = create_metric_plot_obj(
+                #         plot_df,
+                #         BENCHMARK_COLS,
+                #         title="Top Scores and Human Baseline Over Time (from last update)",
+                #     )
+                #     gr.Plot(value=chart, min_width=500)
+
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 

src/about.py

@@ -3,44 +3,38 @@ from enum import Enum
 
 @dataclass
 class Task:
-    phenotype_key: str
-    phenotype_name: str
-    metric_key: str
-    metric_name: str
+    phenotype: str
+    metric: str
 
 
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    task0 = Task("asthma", "Asthma", "auroc", "AUROC")
-    task1 = Task("cataract", "Cataract", "auroc", "AUROC")
-    task2 = Task("diabete", "Diabete", "auroc", "AUROC")
-    task3 = Task("GERD", "GERD", "auroc", "AUROC")
-    task4 = Task("hay-fever-eczema", "Hay-fever & Eczema", "auroc", "AUROC")
-    task5 = Task("hypertension", "Hypertension", "auroc", "AUROC")
-    task6 = Task("major-depression", "Major Depression", "auroc", "AUROC")
-    task7 = Task("migraine", "Migraine", "auroc", "AUROC")
-    task8 = Task("myocardial-infarction", "Myocardial Infarction", "auroc", "AUROC")
-    task9 = Task("osteoarthritis", "Osteoarthritis", "auroc", "AUROC")
-    task10 = Task("pneumonia", "Pneumonia", "auroc", "AUROC")
-    task11 = Task("stroke", "Stroke", "auroc", "AUROC")
-    task12 = Task("asthma", "Asthma", "auprc", "AUPRC")
-    task13 = Task("cataract", "Cataract", "auprc", "AUPRC")
-    task14 = Task("diabete", "Diabete", "auprc", "AUPRC")
-    task15 = Task("GERD", "GERD", "auprc", "AUPRC")
-    task16 = Task("hay-fever-eczema", "Hay-fever & Eczema", "auprc", "AUPRC")
-    task17 = Task("hypertension", "Hypertension", "auprc", "AUPRC")
-    task18 = Task("major-depression", "Major Depression", "auprc", "AUPRC")
-    task19 = Task("migraine", "Migraine", "auprc", "AUPRC")
-    task20 = Task("myocardial-infarction", "Myocardial Infarction", "auprc", "AUPRC")
-    task21 = Task("osteoarthritis", "Osteoarthritis", "auprc", "AUPRC")
-    task22 = Task("pneumonia", "Pneumonia", "auprc", "AUPRC")
-    task23 = Task("stroke", "Stroke", "auprc", "AUPRC")
+    task0 = Task("Asthma", "auroc")
+    task1 = Task("Cataract", "auroc")
+    task2 = Task("Diabetes", "auroc")
+    task3 = Task("GERD", "auroc")
+    task4 = Task("Hay-fever & Eczema", "auroc")
+    task5 = Task("Major depression", "auroc")
+    task6 = Task("Myocardial infarction", "auroc")
+    task7 = Task("Osteoarthritis", "auroc")
+    task8 = Task("Pneumonia", "auroc")
+    task9 = Task("Stroke", "auroc")
+    task10 = Task("Asthma", "auprc")
+    task11 = Task("Cataract", "auprc")
+    task12 = Task("Diabetes", "auprc")
+    task13 = Task("GERD", "auprc")
+    task14 = Task("Hay-fever & Eczema", "auprc")
+    task15 = Task("Major depression", "auprc")
+    task16 = Task("Myocardial infarction", "auprc")
+    task17 = Task("Osteoarthritis", "auprc")
+    task18 = Task("Pneumonia", "auprc")
+    task19 = Task("Stroke", "auprc")
 # ---------------------------------------------------
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">LLM Disease Risk Prediction Leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">LLMs Disease Risk Prediction Leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """

src/display/utils.py

@@ -36,7 +36,7 @@ auto_eval_column_dict.append(["nb_shots", ColumnContent, ColumnContent("#Shots",
 auto_eval_column_dict.append(["average_auroc", ColumnContent, ColumnContent("Average AUROC ⬆️", "number", True)])
 auto_eval_column_dict.append(["average_auprc", ColumnContent, ColumnContent("Average AUPRC ⬆️", "number", True)])
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(generate_column_name(task.value.phenotype_name, task.value.metric_name), "number", displayed_by_default=False, is_task=True, task=task)])
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(generate_column_name(task.value.phenotype, task.value.metric.upper()), "number", displayed_by_default=False, is_task=True, task=task)])
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])

src/envs.py

@@ -1,11 +1,10 @@
 import os
-
 from huggingface_hub import HfApi
 
+
 # Info to change for your repository
 # ----------------------------------
 HF_TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
-
 OWNER = "TemryL" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 

src/leaderboard/read_evals.py

@@ -1,11 +1,9 @@
+import os
 import glob
 import json
-import os
-from dataclasses import dataclass
-
 import dateutil
 import numpy as np
-
+from dataclasses import dataclass
 from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, generate_column_name
 from src.submission.check_validity import is_model_on_hub
@@ -69,12 +67,11 @@ class EvalResult:
         results = {}
         for task in Tasks:
             task = task.value
-            
-            mean = data["results"].get(task.phenotype_key, {}).get("metrics", {}).get("_".join(["mean", task.metric_key]), None)
-            lower = data["results"].get(task.phenotype_key, {}).get("metrics", {}).get("_".join(["lower", task.metric_key]), None)
-            upper = data["results"].get(task.phenotype_key, {}).get("metrics", {}).get("_".join(["upper", task.metric_key]), None)
+            mean = data["results"].get(task.phenotype, {}).get("metrics", {}).get("_".join(["mean", task.metric]), None)
+            lower = data["results"].get(task.phenotype, {}).get("metrics", {}).get("_".join(["lower", task.metric]), None)
+            upper = data["results"].get(task.phenotype, {}).get("metrics", {}).get("_".join(["upper", task.metric]), None)
             formated_score = f"{mean:.2f} ({lower:.2f}-{upper:.2f})" if mean is not None else None
-            results["_".join([task.phenotype_key, task.metric_key])] = formated_score
+            results["_".join([task.phenotype, task.metric])] = formated_score
 
         return self(
             eval_name=f"{org}_{model}_{precision.value.name}_{feature_set}_{nb_shots}",
@@ -121,7 +118,7 @@ class EvalResult:
         }
 
         for task in Tasks:
-            data_dict[generate_column_name(task.value.phenotype_name, task.value.metric_name)] = self.results["_".join([task.value.phenotype_key, task.value.metric_key])]
+            data_dict[generate_column_name(task.value.phenotype, task.value.metric.upper())] = self.results["_".join([task.value.phenotype, task.value.metric])]
         return data_dict
 
 

src/populate.py

@@ -1,11 +1,9 @@
-import json
 import os
-
+import json
 import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results, EvalResult
+from src.leaderboard.read_evals import get_raw_eval_results
 
 
 def get_leaderboard_df(results_path: str, cols: list) -> pd.DataFrame: