select phenotypes and metrics

app.py

@@ -1,4 +1,3 @@
-import subprocess
 import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
@@ -14,7 +13,6 @@ from src.about import (
 )
 from src.display.css_html_js import custom_css
 from src.display.utils import (
-    BENCHMARK_COLS,
     COLS,
     EVAL_COLS,
     EVAL_TYPES,
@@ -24,11 +22,16 @@ from src.display.utils import (
     ModelType,
     fields,
     WeightType,
-    Precision
+    Precision,
+    generate_column_name
 )
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
+from dotenv import load_dotenv
+
+
+load_dotenv()
 
 
 def restart_space():
@@ -50,7 +53,7 @@ except Exception:
     restart_space()
 
 
-raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS)
 leaderboard_df = original_df.copy()
 
 (
@@ -64,6 +67,8 @@ leaderboard_df = original_df.copy()
 def update_table(
     hidden_df: pd.DataFrame,
     columns: list,
+    phenotypes: list,
+    metrics: list,
     type_query: list,
     precision_query: str,
     size_query: list,
@@ -72,7 +77,7 @@ def update_table(
 ):
     filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
     filtered_df = filter_queries(query, filtered_df)
-    df = select_columns(filtered_df, columns)
+    df = select_columns(filtered_df, columns, phenotypes, metrics)
     return df
 
 
@@ -80,14 +85,19 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
 
 
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
+def select_columns(df: pd.DataFrame, columns: list, phenotypes: list, metrics:list) -> pd.DataFrame:
     always_here_cols = [
         AutoEvalColumn.model_type_symbol.name,
         AutoEvalColumn.model.name,
     ]
-    # We use COLS to maintain sorting
+    
+    task_cols = []
+    for phenotype in phenotypes:
+        for metric in metrics:
+            task_cols.append(generate_column_name(phenotype, metric))
+    
     filtered_df = df[
-        always_here_cols + [c for c in COLS if c in df.columns and c in columns] 
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + sorted(task_cols)
     ]
     return filtered_df
 
@@ -147,12 +157,34 @@ with demo:
                             show_label=False,
                             elem_id="search-bar",
                         )
+                    with gr.Row():
+                        with gr.Column(min_width=320):
+                            shown_phenotypes = gr.CheckboxGroup(
+                                choices=sorted(set([
+                                    c.task.value.phenotype_name
+                                    for c in fields(AutoEvalColumn)
+                                    if not c.hidden and not c.never_hidden and c.is_task
+                                ])),
+                                label="Select phenotypes to show",
+                                elem_id="phenotype-select",
+                                interactive=True,
+                            )
+                            shown_metrics = gr.CheckboxGroup(
+                                choices=sorted(set([
+                                    c.task.value.metric_name
+                                    for c in fields(AutoEvalColumn)
+                                    if not c.hidden and not c.never_hidden and c.is_task
+                                ])),
+                                label="Select metrics to show",
+                                elem_id="metric-select",
+                                interactive=True,
+                            )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
                             choices=[
                                 c.name
                                 for c in fields(AutoEvalColumn)
-                                if not c.hidden and not c.never_hidden
+                                if not c.hidden and not c.never_hidden and not c.is_task
                             ],
                             value=[
                                 c.name
@@ -163,12 +195,7 @@ with demo:
                             elem_id="column-select",
                             interactive=True,
                         )
-                    with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=False, label="Show gated/private/deleted models", interactive=True
-                        )
                 with gr.Column(min_width=320):
-                    #with gr.Box(elem_id="box-filter"):
                     filter_columns_type = gr.CheckboxGroup(
                         label="Model types",
                         choices=[t.to_str() for t in ModelType],
@@ -190,6 +217,10 @@ with demo:
                         interactive=True,
                         elem_id="filter-columns-size",
                     )
+                    with gr.Row():
+                        deleted_models_visibility = gr.Checkbox(
+                            value=True, label="Show gated/private/deleted models", interactive=True
+                        )
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
@@ -215,6 +246,8 @@ with demo:
                 [
                     hidden_leaderboard_table_for_search,
                     shown_columns,
+                    shown_phenotypes,
+                    shown_metrics,
                     filter_columns_type,
                     filter_columns_precision,
                     filter_columns_size,
@@ -223,12 +256,14 @@ with demo:
                 ],
                 leaderboard_table,
             )
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
+            for selector in [shown_phenotypes, shown_metrics, shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
                 selector.change(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,
                         shown_columns,
+                        shown_phenotypes,
+                        shown_metrics,
                         filter_columns_type,
                         filter_columns_precision,
                         filter_columns_size,

src/about.py

@@ -3,17 +3,39 @@ from enum import Enum
 
 @dataclass
 class Task:
-    benchmark: str
-    metric: str
-    col_name: str
+    phenotype_key: str
+    phenotype_name: str
+    metric_key: str
+    metric_name: str
 
 
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    task0 = Task("asthma", "Asthma", "auroc", "AUROC")
+    task1 = Task("cataract", "Cataract", "auroc", "AUROC")
+    task2 = Task("diabete", "Diabete", "auroc", "AUROC")
+    task3 = Task("GERD", "GERD", "auroc", "AUROC")
+    task4 = Task("hay-fever-eczema", "Hay-fever & Eczema", "auroc", "AUROC")
+    task5 = Task("hypertension", "Hypertension", "auroc", "AUROC")
+    task6 = Task("major-depression", "Major Depression", "auroc", "AUROC")
+    task7 = Task("migraine", "Migraine", "auroc", "AUROC")
+    task8 = Task("myocardial-infarction", "Myocardial Infarction", "auroc", "AUROC")
+    task9 = Task("osteoarthritis", "Osteoarthritis", "auroc", "AUROC")
+    task10 = Task("pneumonia", "Pneumonia", "auroc", "AUROC")
+    task11 = Task("stroke", "Stroke", "auroc", "AUROC")
+    task12 = Task("asthma", "Asthma", "auprc", "AUPRC")
+    task13 = Task("cataract", "Cataract", "auprc", "AUPRC")
+    task14 = Task("diabete", "Diabete", "auprc", "AUPRC")
+    task15 = Task("GERD", "GERD", "auprc", "AUPRC")
+    task16 = Task("hay-fever-eczema", "Hay-fever & Eczema", "auprc", "AUPRC")
+    task17 = Task("hypertension", "Hypertension", "auprc", "AUPRC")
+    task18 = Task("major-depression", "Major Depression", "auprc", "AUPRC")
+    task19 = Task("migraine", "Migraine", "auprc", "AUPRC")
+    task20 = Task("myocardial-infarction", "Myocardial Infarction", "auprc", "AUPRC")
+    task21 = Task("osteoarthritis", "Osteoarthritis", "auprc", "AUPRC")
+    task22 = Task("pneumonia", "Pneumonia", "auprc", "AUPRC")
+    task23 = Task("stroke", "Stroke", "auprc", "AUPRC")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
@@ -25,7 +47,14 @@ TITLE = """<h1 align="center" id="space-title">OpenHeLM Leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+TODO:
+
+    - Add a description of the leaderboard    
+    - Add class distribution for each phenotype
+    - Potentially a warning when we should not rely on AUROC
+    - Plot of AUROC and AUPRC for each phenotype
+    - Edit about section
+    - Edit submit section (AutoModelForCausalLM)
 """
 
 # Which evaluations are you running? how can people reproduce what you have?

src/display/utils.py

@@ -2,12 +2,15 @@ from dataclasses import dataclass, make_dataclass
 from enum import Enum
 
 import pandas as pd
+from src.about import Task, Tasks
 
-from src.about import Tasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
+def generate_column_name(phenotype_name, metric_name):
+    return f"{phenotype_name} ({metric_name})"
+
 
 # These classes are for user facing column names,
 # to avoid having to change them all around the code
@@ -19,6 +22,8 @@ class ColumnContent:
     displayed_by_default: bool
     hidden: bool = False
     never_hidden: bool = False
+    is_task: bool = False
+    task: Task = None
 
 ## Leaderboard columns
 auto_eval_column_dict = []
@@ -26,9 +31,10 @@ auto_eval_column_dict = []
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["average_auroc", ColumnContent, ColumnContent("Average AUROC ⬆️", "number", True)])
+auto_eval_column_dict.append(["average_auprc", ColumnContent, ColumnContent("Average AUPRC ⬆️", "number", True)])
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(generate_column_name(task.value.phenotype_name, task.value.metric_name), "number", displayed_by_default=False, is_task=True, task=task)])
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
@@ -121,8 +127,6 @@ TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default a
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-
 NUMERIC_INTERVALS = {
     "?": pd.Interval(-1, 0, closed="right"),
     "~1.5": pd.Interval(0, 2, closed="right"),

src/envs.py

@@ -6,12 +6,12 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "TemryL" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+REPO_ID = f"{OWNER}/OpenHeLM-leaderboard"
+QUEUE_REPO = f"{OWNER}/OpenHeLM-requests"
+RESULTS_REPO = f"{OWNER}/OpenHeLM-results"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")

src/leaderboard/read_evals.py

@@ -1,6 +1,5 @@
 import glob
 import json
-import math
 import os
 from dataclasses import dataclass
 
@@ -8,7 +7,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, generate_column_name
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -22,6 +21,7 @@ class EvalResult:
     model: str
     revision: str # commit hash, "" if main
     results: dict
+    raw_data: dict
     precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
     weight_type: WeightType = WeightType.Original # Original or Adapter
@@ -58,7 +58,7 @@ class EvalResult:
         full_model = "/".join(org_and_model)
 
         still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False, token=os.environ.get("TOKEN")
         )
         architecture = "?"
         if model_config is not None:
@@ -70,14 +70,12 @@ class EvalResult:
         results = {}
         for task in Tasks:
             task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
+            
+            mean = data["results"].get(task.phenotype_key, {}).get("_".join(["mean", task.metric_key]), None)
+            lower = data["results"].get(task.phenotype_key, {}).get("_".join(["lower", task.metric_key]), None)
+            upper = data["results"].get(task.phenotype_key, {}).get("_".join(["upper", task.metric_key]), None)
+            formated_score = f"{mean:.2f} ({lower:.2f}-{upper:.2f})" if mean is not None else None
+            results["_".join([task.phenotype_key, task.metric_key])] = formated_score
 
         return self(
             eval_name=result_key,
@@ -85,8 +83,9 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
+            raw_data=data,
+            precision=precision,
+            revision=config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture
         )
@@ -109,7 +108,8 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        average_auroc = np.mean(np.array([d["mean_auroc"] for d in self.raw_data["results"].values() if "mean_auroc" in d.keys()]))
+        average_auprc = np.mean(np.array([d["mean_auprc"] for d in self.raw_data["results"].values() if "mean_auprc" in d.keys()]))
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
@@ -119,7 +119,8 @@ class EvalResult:
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.average_auroc.name: average_auroc,
+            AutoEvalColumn.average_auprc.name: average_auprc,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
@@ -127,8 +128,7 @@ class EvalResult:
         }
 
         for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
+            data_dict[generate_column_name(task.value.phenotype_name, task.value.metric_name)] = self.results["_".join([task.value.phenotype_key, task.value.metric_key])]
         return data_dict
 
 

src/populate.py

@@ -8,17 +8,15 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df.sort_values(by=[AutoEvalColumn.average_auroc.name], ascending=False)
     df = df[cols].round(decimals=2)
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
     return raw_data, df