CHANGES

src/display/utils.py

@@ -28,7 +28,7 @@ for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, field(default_factory=lambda t=_task: ColumnContent(t.value.col_name, "number", True))])
 
 # We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)()
 
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
@@ -80,6 +80,7 @@ class Precision(Enum):
     bfloat16 = ModelDetails("bfloat16")
     Unknown = ModelDetails("?")
 
+    @staticmethod
     def from_str(precision):
         if precision in ["torch.float16", "float16"]:
             return Precision.float16

src/envs.py

@@ -17,5 +17,7 @@ CACHE_PATH=os.getenv("HF_HOME", ".")
 
 # Local caches
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+QUEUE_REPO = f"{OWNER}/requests"
 
 API = HfApi(token=TOKEN)

src/populate.py

@@ -1,7 +1,17 @@
-from src.leaderboard.read_evals import get_raw_eval_results
+import json
+import os
+
 import pandas as pd
 
-""" calls get_raw_eval_results function from our read_evals.py file to get the DataFrame"""
+from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.leaderboard.read_evals import get_raw_eval_results
+
+
+def has_no_nan_values(df, cols):
+    if not cols:
+        return [True] * len(df)
+    return df[cols].notna().all(axis=1)
+
 
 def get_leaderboard_df(results_path: str, requests_path: str = None, cols: list = None, benchmark_cols: list = None) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
@@ -9,16 +19,17 @@ def get_leaderboard_df(results_path: str, requests_path: str = None, cols: list
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
+    df = df.sort_values(by=[AutoEvalColumn.technique.name], ascending=True)
+    if cols:
+        df = df[[c for c in cols if c in df.columns]].round(decimals=2)
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
+    if benchmark_cols:
+        df = df[has_no_nan_values(df, benchmark_cols)]
     return df
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
+    """Creates the different dataframes for the evaluation queues"""
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []
 
@@ -28,19 +39,21 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             with open(file_path) as fp:
                 data = json.load(fp)
 
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.model.name] = data["model"]
             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
 
             all_evals.append(data)
         elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(os.path.join(save_path, entry, e)) and not e.startswith(".")]
+            sub_entries = [
+                e for e in os.listdir(f"{save_path}/{entry}")
+                if os.path.isfile(os.path.join(save_path, entry, e)) and not e.startswith(".")
+            ]
             for sub_entry in sub_entries:
                 file_path = os.path.join(save_path, entry, sub_entry)
                 with open(file_path) as fp:
                     data = json.load(fp)
 
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.model.name] = data["model"]
                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
                 all_evals.append(data)