Add single column leaderboard

app.py

@@ -20,6 +20,7 @@ from src.display.utils import (
     EVAL_TYPES,
     AutoEvalColumn,
     singletable_AutoEvalColumn,
+    singlecolumn_AutoEvalColumn,
     ModelType,
     fields,
     # WeightType,
@@ -50,7 +51,7 @@ except Exception:
     restart_space()
 
 
-SINGLETABLE_LEADERBOARD_DF, MULTITABLE_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
+SINGLECOLUMN_LEADERBOARD_DF, SINGLETABLE_LEADERBOARD_DF, MULTITABLE_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
 
 (
     finished_eval_queue_df,
@@ -111,6 +112,28 @@ def init_singletable_leaderboard(dataframe):
         interactive=False,
     )
 
+def init_singlecolumn_leaderboard(dataframe):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(singlecolumn_AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=[c.name for c in fields(singlecolumn_AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(singlecolumn_AutoEvalColumn) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[singlecolumn_AutoEvalColumn.model.name], # AutoEvalColumn.license.name],
+        hide_columns=[c.name for c in fields(singlecolumn_AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(singlecolumn_AutoEvalColumn.dataset.name, type="checkboxgroup", label="Datasets"),
+            ColumnFilter(singlecolumn_AutoEvalColumn.table.name, type="checkboxgroup", label="Tables"),
+            ColumnFilter(singlecolumn_AutoEvalColumn.model.name, type="checkboxgroup", label="Models"),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+
 
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -124,12 +147,15 @@ with demo:
         with gr.TabItem("🏅 SingleTable", elem_id="syntherela-benchmark-tab-table", id=1):
             singletable_leaderboard = init_singletable_leaderboard(SINGLETABLE_LEADERBOARD_DF)
 
+        with gr.TabItem("🏅 SingleColumn", elem_id="syntherela-benchmark-tab-table", id=2):
+            singlecolumn_leaderboard = init_singlecolumn_leaderboard(SINGLECOLUMN_LEADERBOARD_DF)
+
 
 
-        with gr.TabItem("📝 About", elem_id="syntherela-benchmark-tab-table", id=2):
+        with gr.TabItem("📝 About", elem_id="syntherela-benchmark-tab-table", id=3):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="syntherela-benchmark-tab-table", id=3):
+        with gr.TabItem("🚀 Submit here! ", elem_id="syntherela-benchmark-tab-table", id=4):
             with gr.Column():
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")

src/about.py

@@ -27,6 +27,24 @@ class SingleTableTasks(Enum):
     # SingleTableDetection-XGBClassifier
     task_3 = Task("single-table", "SingleTableDetection-XGBClassifier", "SingleTableDetection-XGBClassifier ⬇️")
 
+class SingleColumnTasks(Enum):
+    # ChiSquareTest
+    task_0 = Task("single-column", "ChiSquareTest", "ChiSquareTest ⬇️")
+    # HellingerDistance
+    task_1 = Task("single-column", "HellingerDistance", "HellingerDistance ⬇️")
+    # JensenShannonDistance
+    task_2 = Task("single-column", "JensenShannonDistance", "JensenShannonDistance ⬇️")
+    # KolmogorovSmirnovTest
+    task_3 = Task("single-column", "KolmogorovSmirnovTest", "KolmogorovSmirnovTest ⬇️")
+    # SingleColumnDetection-LogisticRegression
+    task_4 = Task("single-column", "SingleColumnDetection-LogisticRegression", "SingleColumnDetection-LogisticRegression ⬇️")
+    # SingleColumnDetection-XGBClassifier
+    task_5 = Task("single-column", "SingleColumnDetection-XGBClassifier", "SingleColumnDetection-XGBClassifier ⬇️")
+    # TotalVariationDistance
+    task_6 = Task("single-column", "TotalVariationDistance", "TotalVariationDistance ⬇️")
+    # WassersteinDistance
+    task_7 = Task("single-column", "WassersteinDistance", "WassersteinDistance ⬇️")
+
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
 

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks, SingleTableTasks
+from src.about import Tasks, SingleTableTasks, SingleColumnTasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -55,6 +55,21 @@ for task in SingleTableTasks:
 
 singletable_AutoEvalColumn = make_dataclass("AutoEvalColumn", singletable_auto_eval_column_dict, frozen=True)
 
+# SINGLE COLUMN
+
+singlecolumn_auto_eval_column_dict = []
+# Init
+singlecolumn_auto_eval_column_dict.append(["dataset", ColumnContent, ColumnContent("Dataset", "str", True, never_hidden=True)])
+singlecolumn_auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+singlecolumn_auto_eval_column_dict.append(["table", ColumnContent, ColumnContent("Table", "str", True, never_hidden=True)])
+#Scores
+for task in SingleColumnTasks:
+    singlecolumn_auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+
+singlecolumn_AutoEvalColumn = make_dataclass("AutoEvalColumn", singlecolumn_auto_eval_column_dict, frozen=True)
+
+
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column

src/populate.py

@@ -7,7 +7,7 @@ import numpy as np
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
-from src.about import Tasks, SingleTableTasks
+from src.about import Tasks, SingleTableTasks, SingleColumnTasks
 
 
 # def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
@@ -43,12 +43,13 @@ def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> p
                 all_data_json.append(data)
 
     multi_table_metrics = [task.value.col_name for task in Tasks]
-
     single_table_metrics = [task.value.col_name for task in SingleTableTasks]
+    single_column_metrics = [task.value.col_name for task in SingleColumnTasks]
 
     # create empty dataframe with the columns multi_table_metrics
     multitable_df = pd.DataFrame(columns=["Dataset", "Model"] + multi_table_metrics)
     singletable_df = pd.DataFrame(columns=["Dataset", "Model"] + single_table_metrics)
+    singlecolumn_df = pd.DataFrame(columns=["Dataset", "Table", "Model"] + single_column_metrics)
 
     # iterate through all json files and add the data to the dataframe
     for data in all_data_json:
@@ -86,7 +87,38 @@ def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> p
                 singletable_row[metric] = np.nan
         singletable_df = pd.concat([singletable_df, pd.DataFrame([singletable_row])], ignore_index=True)
 
-    return singletable_df, multitable_df
+        singlecolumn_row = {"Dataset": dataset, "Model": model, "Table": ""}
+        # insert row
+        for metric in single_column_metrics:
+            stripped_metric = strip_emoji(metric)
+            if stripped_metric in data["single_column_metrics"]:
+                for table in data["single_column_metrics"][stripped_metric].keys():
+                    # check if row where dataset = dataset, model = model, table = table exists
+                    if singlecolumn_df[
+                        (singlecolumn_df["Dataset"] == dataset) & 
+                        (singlecolumn_df["Model"] == model) & 
+                        (singlecolumn_df["Table"] == table)
+                    ].empty:
+                        singlecolumn_row = {"Dataset": dataset, "Model": model, "Table": table}
+                        singlecolumn_df = pd.concat([singlecolumn_df, pd.DataFrame([singlecolumn_row])], ignore_index=True)
+
+                    metric_values = []
+                    for column in data["single_column_metrics"][stripped_metric][table].keys():
+                        if "accuracy" in data["single_column_metrics"][stripped_metric][table][column]:
+                            metric_values.append(data["single_column_metrics"][stripped_metric][table][column]["accuracy"])
+                        if "value" in data["single_column_metrics"][stripped_metric][table][column]:
+                            metric_values.append(data["single_column_metrics"][stripped_metric][table][column]["value"])
+                        if "statistic" in data["single_column_metrics"][stripped_metric][table][column]:
+                            metric_values.append(data["single_column_metrics"][stripped_metric][table][column]["statistic"])
+
+                    # save np.mean(metric_values).round(decimals=2) to singlecolumn_df where dataset = dataset, model = model, table = table
+                    singlecolumn_df.loc[
+                        (singlecolumn_df["Dataset"] == dataset) & 
+                        (singlecolumn_df["Model"] == model) & 
+                        (singlecolumn_df["Table"] == table), metric] = np.mean(metric_values).round(decimals=2)
+            
+
+    return singlecolumn_df, singletable_df, multitable_df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]: