Refactor AutoEvalColumn usage for consistency and clarity; add debug prints for NaN checks in formatting functions

app.py

@@ -62,26 +62,26 @@ def init_leaderboard(dataframe):
         raise ValueError("Leaderboard DataFrame is empty or None.")
     return Leaderboard(
         value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
+        datatype=[c.type for c in fields(AutoEvalColumn())],
         select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            default_selection=[c.name for c in fields(AutoEvalColumn()) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn()) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        search_columns=[AutoEvalColumn().model.name, AutoEvalColumn().license.name],
+        hide_columns=[c.name for c in fields(AutoEvalColumn()) if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            ColumnFilter(AutoEvalColumn().model_type.name, type="checkboxgroup", label="Model types"),
+            ColumnFilter(AutoEvalColumn().precision.name, type="checkboxgroup", label="Precision"),
             ColumnFilter(
-                AutoEvalColumn.params.name,
+                AutoEvalColumn().params.name,
                 type="slider",
                 min=0.01,
                 max=150,
                 label="Select the number of parameters (B)",
             ),
             ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+                AutoEvalColumn().still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
             ),
         ],
         bool_checkboxgroup_label="Hide models",

src/display/formatting.py

@@ -20,6 +20,8 @@ def styled_message(message):
 
 
 def has_no_nan_values(df, columns):
+    print(df.columns)
+    print(columns)
     return df[columns].notna().all(axis=1)
 
 

src/display/utils.py

@@ -99,7 +99,7 @@ class Precision(Enum):
         return Precision.Unknown
 
 # Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+COLS = [c.name for c in fields(AutoEvalColumn()) if not c.hidden]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]

src/leaderboard/read_evals.py

@@ -110,20 +110,21 @@ class EvalResult:
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        AutoEvalColumnInstance = AutoEvalColumn()
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            AutoEvalColumnInstance.precision.name: self.precision.value.name,
+            AutoEvalColumnInstance.model_type.name: self.model_type.value.name,
+            AutoEvalColumnInstance.model_type_symbol.name: self.model_type.value.symbol,
+            AutoEvalColumnInstance.weight_type.name: self.weight_type.value.name,
+            AutoEvalColumnInstance.architecture.name: self.architecture,
+            AutoEvalColumnInstance.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumnInstance.revision.name: self.revision,
+            AutoEvalColumnInstance.average.name: average,
+            AutoEvalColumnInstance.license.name: self.license,
+            AutoEvalColumnInstance.likes.name: self.likes,
+            AutoEvalColumnInstance.params.name: self.num_params,
+            AutoEvalColumnInstance.still_on_hub.name: self.still_on_hub,
         }
 
         for task in Tasks:
@@ -157,19 +158,10 @@ def get_request_file_for_model(requests_path, model_name, precision):
 def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []
-
     for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
+        # Only process .json files
+        json_files = [f for f in files if f.endswith(".json")]
+        for file in json_files:
             model_result_filepaths.append(os.path.join(root, file))
 
     eval_results = {}
@@ -190,7 +182,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         try:
             v.to_dict() # we test if the dict version is complete
             results.append(v)
-        except KeyError:  # not all eval values present
+        except KeyError as e:  # not all eval values present
+            print(e)
             continue
-
     return results

src/populate.py

@@ -12,13 +12,9 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
     all_data_json = [v.to_dict() for v in raw_data]
-
     df = pd.DataFrame.from_records(all_data_json)
-    print("Columns:", df.columns.tolist())
-
     df = df.sort_values(by=[AutoEvalColumn().average.name], ascending=False)
     df = df[cols].round(decimals=2)
-
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
     return df
@@ -39,14 +35,13 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
 
             all_evals.append(data)
-        elif ".md" not in entry:
+        elif ".md" not in entry and os.path.isdir(os.path.join(save_path, entry)):
             # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
+            sub_entries = [e for e in os.listdir(os.path.join(save_path, entry)) if os.path.isfile(os.path.join(save_path, entry, e)) and not e.startswith(".")]
             for sub_entry in sub_entries:
                 file_path = os.path.join(save_path, entry, sub_entry)
                 with open(file_path) as fp:
                     data = json.load(fp)
-
                 data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
                 all_evals.append(data)