filter feature set

app.py

@@ -70,6 +70,7 @@ def update_table(
     columns: list,
     phenotypes: list,
     metrics: list,
+    feature_sets: list,
     nb_shots: list,
     type_query: list,
     precision_query: str,
@@ -77,7 +78,7 @@ def update_table(
     show_deleted: bool,
     query: str,
 ):
-    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted, nb_shots)
+    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted, feature_sets, nb_shots)
     filtered_df = filter_queries(query, filtered_df)
     df = select_columns(filtered_df, columns, phenotypes, metrics)
     return df
@@ -91,6 +92,7 @@ def select_columns(df: pd.DataFrame, columns: list, phenotypes: list, metrics:li
     always_here_cols = [
         AutoEvalColumn.model_type_symbol.name,
         AutoEvalColumn.model.name,
+        AutoEvalColumn.feature_set.name,
         AutoEvalColumn.nb_shots.name,
     ]
     
@@ -125,7 +127,7 @@ def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
 
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool, nb_shots: list) -> pd.DataFrame:
+    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool, feature_sets: list, nb_shots: list) -> pd.DataFrame:
     # Show all models
     if show_deleted:
         filtered_df = df
@@ -137,6 +139,7 @@ def filter_models(
     filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
     if -1 not in nb_shots:
         filtered_df = filtered_df.loc[df[AutoEvalColumn.nb_shots.name].isin(nb_shots)]
+    filtered_df = filtered_df.loc[df[AutoEvalColumn.feature_set.name].isin(feature_sets)]
     
     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
@@ -155,6 +158,12 @@ with demo:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
                     with gr.Row():
                         with gr.Column(min_width=320):
                             shown_phenotypes = gr.CheckboxGroup(
@@ -173,6 +182,11 @@ with demo:
                                     for c in fields(AutoEvalColumn)
                                     if not c.hidden and not c.never_hidden and c.is_task
                                 ])),
+                                value=sorted(set([
+                                    c.task.value.metric_name
+                                    for c in fields(AutoEvalColumn)
+                                    if not c.hidden and not c.never_hidden and c.is_task
+                                ])),
                                 label="Select metrics to show",
                                 elem_id="metric-select",
                                 interactive=True,
@@ -193,18 +207,23 @@ with demo:
                             elem_id="column-select",
                             interactive=True,
                         )
-                with gr.Column(min_width=320):
                     with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
-                            show_label=False,
-                            elem_id="search-bar",
+                        deleted_models_visibility = gr.Checkbox(
+                            value=True, label="Show gated/private/deleted models", interactive=True
                         )
+                with gr.Column(min_width=320):
                     with gr.Column(min_width=320):
+                        filter_features = gr.CheckboxGroup(
+                            label="Features Set",
+                            choices=[("Baseline (age, sex, BMI)", "baseline"), ("Expanded (age, sex, BMI, HDL, LDL, total-cholesterol, triglycerides, diastolic-blood-pressure, smoking-status, snoring, insomnia, daytime-napping, sleep-duration, chronotype)", "expanded")],
+                            value=["baseline"],
+                            interactive=True,
+                            elem_id="filter-feature-set",
+                        )
                         filter_nb_shots = gr.CheckboxGroup(
                             label="Number of shots",
                             choices=[("Zero-shot", 0), ("10-shot", 10), ("All", -1)],
-                            value=[-1],
+                            value=[0],
                             interactive=True,
                             elem_id="filter-nb-shots",
                         )
@@ -229,10 +248,6 @@ with demo:
                             interactive=True,
                             elem_id="filter-columns-size",
                         )
-                    with gr.Row():
-                        deleted_models_visibility = gr.Checkbox(
-                            value=True, label="Show gated/private/deleted models", interactive=True
-                        )
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
@@ -274,7 +289,7 @@ with demo:
                 ],
                 leaderboard_table,
             )
-            for selector in [shown_phenotypes, shown_metrics, shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility, filter_nb_shots]:
+            for selector in [shown_phenotypes, shown_metrics, shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility, filter_nb_shots, filter_features]:
                 selector.change(
                     update_table,
                     [
@@ -282,6 +297,7 @@ with demo:
                         shown_columns,
                         shown_phenotypes,
                         shown_metrics,
+                        filter_features,
                         filter_nb_shots,
                         filter_columns_type,
                         filter_columns_precision,

src/display/utils.py

@@ -30,6 +30,7 @@ auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["feature_set", ColumnContent, ColumnContent("Feature Set", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["nb_shots", ColumnContent, ColumnContent("#Shots", "number", True, never_hidden=True)])
 #Scores
 auto_eval_column_dict.append(["average_auroc", ColumnContent, ColumnContent("Average AUROC ⬆️", "number", True)])

src/leaderboard/read_evals.py

@@ -22,7 +22,8 @@ class EvalResult:
     revision: str # commit hash, "" if main
     results: dict
     raw_data: dict
-    nb_shots: int = 0
+    nb_shots: int
+    feature_set: str
     precision: Precision = Precision.Unknown
     model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
     weight_type: WeightType = WeightType.Original # Original or Adapter
@@ -46,8 +47,8 @@ class EvalResult:
         model = full_model.split("/")[1]
         precision = Precision.from_str(config.get("precision"))
         revision = config.get("revision", "")
-        feature_set = config.get("feature_set", "Unknown")
         nb_shots = config.get("nb_shots", None)
+        feature_set = config.get("feature_set", None)
         model_type = ModelType.from_str(config.get("model_type", ""))
         weight_type = WeightType[config.get("weight_type", "Original")]
         license = config.get("license", "?")
@@ -83,6 +84,7 @@ class EvalResult:
             results=results,
             raw_data=data,
             nb_shots=nb_shots,
+            feature_set=feature_set,
             precision=precision,
             revision=revision,
             still_on_hub=still_on_hub,
@@ -101,6 +103,7 @@ class EvalResult:
         average_auprc = np.mean(np.array([d["metrics"]["mean_auprc"] for d in self.raw_data["results"].values() if "mean_auprc" in d["metrics"].keys()]))
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.feature_set.name: self.feature_set,
             AutoEvalColumn.nb_shots.name: self.nb_shots,
             AutoEvalColumn.precision.name: self.precision.value.name,
             AutoEvalColumn.model_type.name: self.model_type.value.name,