Update src/helper.py

src/helper.py

@@ -5,7 +5,7 @@ import json
 import numpy as np
 from statistics import mean
 import re
-from datasets import load_dataset
+from datasets import load_dataset, concatenate_datasets
 import os
 from collections import defaultdict
 from src.envs import API, SAHARA_DATA, SAHARA_RESULTS
@@ -53,13 +53,23 @@ TASK_TO_CLUSTER_MAP = {
 # ===== Authenticate and Load Data From Private HF Repo =====
 
 def load_private_leaderboard_df():
+
+    all_repo_files = API.list_repo_files(repo_id=SAHARA_DATA, repo_type="dataset")
+    folder_path = "data/users/" 
+    jsonl_files_in_folder = [
+        f for f in all_repo_files 
+        if f.startswith(folder_path) and f.endswith(".jsonl")
+    ]
+    jsonl_files_in_folder.append(SAHARA_RESULTS)
+    print("++++++",jsonl_files_in_folder)
     ds = load_dataset(
         path=SAHARA_DATA,
         name=None,
-        data_files=SAHARA_RESULTS,
+        data_files=jsonl_files_in_folder,
         split="train",
         download_mode="force_redownload"
     )
+    print(">>>>>>>", ds)
     return ds.to_pandas()
 metrics_list={
     'bleu_1k':'spBleu<sup>1K</sup>',
@@ -155,9 +165,9 @@ def compare_models(model_1_name, model_2_name):
         if pd.isna(d):
             return "---"
         if d > 0.001:  # Model 1 is better
-            return f"<span style='color:green; font-weight:bold;'>+{d:.2f}</span>"
+            return f"<span style='color:green !important; font-weight:bold !important;'>+{d:.2f}</span>"
         elif d < -0.001:  # Model 2 is better
-            return f"<span style='color:red; font-weight:bold;'>{d:.2f}</span>"
+            return f"<span style='color:red !important; font-weight:bold !important;'>{d:.2f}</span>"
         else:
             return f"{d:.2f}"