Refactor get_leaderboard_df to handle multiple subsets and improve column selection and hiding

app.py

@@ -64,7 +64,7 @@ except Exception:
     restart_space()
 
 
-LEADERBOARD_DF = get_leaderboard_df(RESULTS_REPO, EVAL_RESULTS_PATH)
+LEADERBOARD_DF = get_leaderboard_df(RESULTS_REPO)
 
 
 def init_leaderboard(dataframes):

src/populate.py

@@ -6,17 +6,17 @@ import pandas as pd
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
 
-from datasets import load_dataset
+from datasets import load_dataset, get_dataset_config_names
 
 
-def get_leaderboard_df(results_repo, results_path):
-    subsets = load_dataset(results_repo, split='all', cache_dir=results_path)
+def get_leaderboard_df(results_repo):
+    subsets = get_dataset_config_names(results_repo)
     subset_dfs = {}
-    print("Subsets:", subsets)
-    for subset_name, subset_data in subsets.items():
+    for subset in subsets:
+        subset_data = load_dataset(results_repo, subset)
         subset_df = subset_data.to_pandas()
         subset_df = subset_df.sort_values(by="Total", ascending=False)
         subset_df = subset_df.round(2)  # Round all numeric columns to two decimal places
         print(subset_df)
-        subset_dfs[subset_name] = subset_df
+        subset_dfs[subset] = subset_df
     return subset_dfs