add dashboard

src/display/formatting.py

@@ -1,12 +1,8 @@
-import os
-from datetime import datetime, timezone
+# utility functions for formatting text and data for display in the leaderboard
 
-from huggingface_hub import HfApi
-from huggingface_hub.hf_api import ModelInfo
+from src.envs import DASHBOARD_LINK
 
 
-API = HfApi()
-
 def model_hyperlink(link, model_name):
     return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
 
@@ -16,6 +12,12 @@ def make_clickable_model(model_name):
     return model_hyperlink(link, model_name)
 
 
+def model_dashboard_hyperlink(model_name):
+    link = DASHBOARD_LINK.format(model_id=model_name)
+    html_link = f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">show in 📊</a>'
+    return html_link
+
+
 def styled_error(error):
     return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
 

src/display/utils.py

@@ -1,7 +1,8 @@
 from dataclasses import dataclass, make_dataclass
 from enum import Enum
+from typing import Any
 
-import pandas as pd
+import pandas as pd # type: ignore
 
 from src.display.about import Tasks
 
@@ -22,26 +23,28 @@ class ColumnContent:
     dummy: bool = False
 
 ## Leaderboard columns
-auto_eval_column_dict = []
+auto_eval_column_dict: list[tuple[str, type, Any]] = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(("model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)))
+auto_eval_column_dict.append(("model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)))
+# Scores
+auto_eval_column_dict.append(("average", ColumnContent, ColumnContent("Average ⬆️", "number", True)))
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_dict.append((task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)))
+# Dashboard
+auto_eval_column_dict.append(("dashboard_link", ColumnContent, ColumnContent("Dashboard", "markdown", False)))
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(("model_type", ColumnContent, ColumnContent("Type", "str", False)))
+auto_eval_column_dict.append(("architecture", ColumnContent, ColumnContent("Architecture", "str", False)))
+auto_eval_column_dict.append(("weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)))
+auto_eval_column_dict.append(("precision", ColumnContent, ColumnContent("Precision", "str", False)))
+auto_eval_column_dict.append(("license", ColumnContent, ColumnContent("Hub License", "str", False)))
+auto_eval_column_dict.append(("params", ColumnContent, ColumnContent("#Params (B)", "number", False)))
+auto_eval_column_dict.append(("likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)))
+auto_eval_column_dict.append(("still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)))
+auto_eval_column_dict.append(("revision", ColumnContent, ColumnContent("Model sha", "str", False, False)))
 # Dummy column for the search bar (hidden by the custom CSS)
-auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
+auto_eval_column_dict.append(("dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)))
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
@@ -53,7 +56,7 @@ class EvalQueueColumn:  # Queue column
     revision = ColumnContent("revision", "str", True)
     private = ColumnContent("private", "bool", True)
     precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
+    weight_type = ColumnContent("weight_type", "str", True)
     status = ColumnContent("status", "str", True)
 
 ## All the model information that we might need

src/envs.py

@@ -1,6 +1,6 @@
 import os
 
-from huggingface_hub import HfApi
+from huggingface_hub import HfApi  # type: ignore
 
 # clone / pull the lmeh eval data
 TOKEN = os.environ.get("TOKEN", None)
@@ -11,9 +11,11 @@ REPO_ID = f"{OWNER}/open_cot_leaderboard"
 QUEUE_REPO = f"{DATA_OWNER}/cot-leaderboard-requests"
 RESULTS_REPO = f"{DATA_OWNER}/cot-leaderboard-results"
 
-
 CACHE_PATH=os.getenv("HF_HOME", ".")
 
+# Dashboard
+DASHBOARD_LINK = "https://huggingface.co/cot-leaderboard/open-cot-dashboard?model={model_id}"
+
 # Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")

src/leaderboard/read_evals.py

@@ -1,13 +1,12 @@
 import glob
 import json
-import math
 import os
 from dataclasses import dataclass
 
-import dateutil
+import dateutil  # type: ignore
 import numpy as np
 
-from src.display.formatting import make_clickable_model
+from src.display.formatting import make_clickable_model, model_dashboard_hyperlink
 from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
 from src.submission.check_validity import is_model_on_hub
 from src.envs import TOKEN
@@ -117,6 +116,7 @@ class EvalResult:
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.dashboard_link.name: model_dashboard_hyperlink(self.full_model),
             AutoEvalColumn.dummy.name: self.full_model,
             AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
@@ -172,7 +172,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
-    eval_results = {}
+    eval_results: dict[str, EvalResult] = {}
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)