udpate

src/leaderboard/read_evals.py

@@ -74,14 +74,15 @@ class EvalResult:
             # We average all scores of a given metric (not all metrics are present in all files)
             accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
             if accs.size == 0 or any([acc is None for acc in accs]):
-                mean_acc = None
+                mean_acc = 0.0
+                # todo: None
             else:
                 mean_acc = np.nanmean(accs) * 100.0
             # if accs.size == 0:
             #     continue
             # mean_acc = np.mean(accs) * 100.0
             results[task.benchmark] = mean_acc
-        print(results)
+        print(full_model, results)
         return self(
             eval_name=result_key,
             full_model=full_model,
@@ -112,7 +113,8 @@ class EvalResult:
 
     def to_dict(self, task_class):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        scores = [v for v in self.results.values() if v is not None]
+        # ignore uncertainty for overall calculation
+        scores = [v for k, v in self.results.values() if v is not None and k != 'uncertainty']
         average = sum(scores) / len(scores)
         # average = sum([v for v in self.results.values() if v is not None]) / len(task_class)
         data_dict = {
@@ -184,8 +186,8 @@ def get_raw_eval_results(results_path: str, requests_path: str, task_class) -> l
         # Store results of same eval together
         eval_name = eval_result.eval_name
         if eval_name in eval_results.keys():
-            # eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items()})
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+            # eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items()})
         else:
             eval_results[eval_name] = eval_result