fix: fix the mean calculation for NAN values

src/read_evals.py

@@ -7,7 +7,6 @@ from typing import List
 import pandas as pd
 
 from src.benchmarks import get_safe_name
-from src.display.formatting import has_no_nan_values
 from src.display.utils import (
     COL_NAME_RERANKING_MODEL,
     COL_NAME_RETRIEVAL_MODEL,
@@ -27,6 +26,13 @@ from src.display.utils import (
 from src.display.formatting import make_clickable_model
 
 
+def calculate_mean(row):
+    if pd.isna(row).any():
+        return 0
+    else:
+        return row.mean()
+
+
 @dataclass
 class EvalResult:
     """
@@ -203,7 +209,7 @@ def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -
     _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
 
     # calculate the average score for selected benchmarks
-    df[COL_NAME_AVG] = df[list(_benchmark_cols)].mean(axis=1).round(decimals=2)
+    df[COL_NAME_AVG] = df[list(_benchmark_cols)].apply(calculate_mean, axis=1).round(decimals=2)
     df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
     df.reset_index(inplace=True, drop=True)
 

src/utils.py

@@ -11,7 +11,7 @@ from src.display.formatting import styled_message, styled_error
 from src.display.utils import COLS_QA, TYPES_QA, COLS_LONG_DOC, TYPES_LONG_DOC, COL_NAME_RANK, COL_NAME_AVG, \
     COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_IS_ANONYMOUS, get_default_auto_eval_column_dict
 from src.envs import API, SEARCH_RESULTS_REPO
-from src.read_evals import FullEvalResult, get_leaderboard_df
+from src.read_evals import FullEvalResult, get_leaderboard_df, calculate_mean
 
 
 def filter_models(df: pd.DataFrame, reranking_query: list) -> pd.DataFrame:
@@ -100,6 +100,7 @@ def select_columns(df: pd.DataFrame, domain_query: list, language_query: list, t
     # We use COLS to maintain sorting
     filtered_df = df[FIXED_COLS + selected_cols]
     filtered_df[COL_NAME_AVG] = filtered_df[selected_cols].mean(axis=1, numeric_only=True).round(decimals=2)
+    filtered_df[COL_NAME_AVG] = filtered_df[selected_cols].apply(calculate_mean, axis=1).round(decimals=2)
     filtered_df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
     filtered_df.reset_index(inplace=True, drop=True)
     filtered_df[COL_NAME_RANK] = filtered_df[COL_NAME_AVG].rank(ascending=False, method="min")