update

app.py

@@ -30,11 +30,7 @@ from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 # from src.submission.check_validity import already_submitted_models
 # from src.tools.collections import update_collections
-from src.tools.plots import (
-    create_metric_plot_obj,
-    create_plot_df,
-    create_scores_df,
-)
+# from src.tools.plots import create_metric_plot_obj, create_plot_df, create_scores_df
 
 
 def restart_space():

src/display/utils.py

@@ -62,6 +62,7 @@ auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
@@ -72,6 +73,18 @@ class EvalQueueColumn:  # Queue column
     status = ColumnContent("status", "str", True)
 
 
+# Define the human baselines
+human_baseline_row = {
+    AutoEvalColumn.model.name: "<p>Human performance</p>",
+    AutoEvalColumn.revision.name: "N/A",
+    AutoEvalColumn.precision.name: None,
+    AutoEvalColumn.average.name: 100.0,
+    AutoEvalColumn.nqopen.name: 100.0,
+    AutoEvalColumn.triviaqa.name: 100.0,
+    AutoEvalColumn.dummy.name: "human_baseline",
+    AutoEvalColumn.model_type.name: "",
+}
+
 @dataclass
 class ModelDetails:
     name: str

src/tools/plots.py

@@ -93,7 +93,8 @@ def create_metric_plot_obj(df: pd.DataFrame, metrics: list[str], title: str) ->
     df = df[df["task"].isin(metrics)]
 
     # Filter the human baselines based on the specified metrics
-    # filtered_human_baselines = {k: v for k, v in HUMAN_BASELINE.items() if k in metrics}
+    from src.display.utils import human_baseline_row as HUMAN_BASELINE
+    filtered_human_baselines = {k: v for k, v in HUMAN_BASELINE.items() if k in metrics}
 
     # Create a line figure using plotly express with specified markers and custom data
     fig = px.line(
@@ -128,6 +129,21 @@ def create_metric_plot_obj(df: pd.DataFrame, metrics: list[str], title: str) ->
     for trace in fig.data:
         metric_color_mapping[trace.name] = trace.line.color
 
+    # Iterate over filtered human baselines and add horizontal lines to the figure
+    for metric, value in filtered_human_baselines.items():
+        color = metric_color_mapping.get(metric, "blue")  # Retrieve color from mapping; default to blue if not found
+        location = "top left" if metric == "HellaSwag" else "bottom left"  # Set annotation position
+        # Add horizontal line with matched color and positioned annotation
+        fig.add_hline(
+            y=value,
+            line_dash="dot",
+            annotation_text=f"{metric} human baseline",
+            annotation_position=location,
+            annotation_font_size=10,
+            annotation_font_color=color,
+            line_color=color,
+        )
+
     return fig