consolidate

src/display/utils.py

@@ -28,8 +28,8 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
-    # All perplexity scores show with ⬇️ since lower is better
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(f"{task.value.col_name} ⬇️", "number", True)])
+    # Use exact column name from Tasks
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])

src/leaderboard/read_evals.py

@@ -78,9 +78,11 @@ class EvalResult:
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         # Calculate average, handling perplexity (lower is better)
         scores = []
+        perplexity_score = None
         for task in Tasks:
             if task.value.benchmark in self.results:
                 score = self.results[task.value.benchmark]
+                perplexity_score = score  # Save the raw score
                 # Convert perplexity to a 0-100 scale where lower perplexity = higher score
                 # Using a log scale since perplexity can vary widely
                 # Cap at 100 for very low perplexity and 0 for very high perplexity
@@ -106,14 +108,11 @@ class EvalResult:
             AutoEvalColumn.likes.name: 0,  # Default likes
         }
 
-        for task in Tasks:
-            benchmark = task.value.benchmark
-            if benchmark in self.results:
-                score = self.results[benchmark]
-                # Store original perplexity score (lower is better)
-                data_dict[f"{task.value.col_name} ⬇️"] = score
-            else:
-                data_dict[f"{task.value.col_name} ⬇️"] = None
+        # Add perplexity score with the exact column name from Tasks
+        if perplexity_score is not None:
+            data_dict[Tasks.task0.value.col_name] = perplexity_score
+        else:
+            data_dict[Tasks.task0.value.col_name] = None
 
         return data_dict
 
@@ -131,22 +130,27 @@ def get_raw_eval_results(results_path: str) -> list[EvalResult]:
 
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        try:
+            # Creation of result
+            eval_result = EvalResult.init_from_json_file(model_result_filepath)
 
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
+            # Store results of same eval together
+            eval_name = eval_result.eval_name
+            if eval_name in eval_results.keys():
+                eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+            else:
+                eval_results[eval_name] = eval_result
+        except Exception as e:
+            print(f"Error processing result file {model_result_filepath}: {e}")
+            continue
 
     results = []
     for v in eval_results.values():
         try:
             v.to_dict() # we test if the dict version is complete
             results.append(v)
-        except KeyError:  # not all eval values present
+        except KeyError as e:  # not all eval values present
+            print(f"Error converting result to dict: {e}")
             continue
 
     return results