[Add, Fix] change to CAPA, fix error in dataloading

app.py

@@ -110,7 +110,7 @@ with gr.Blocks(title="LLM Similarity Analyzer") as demo:
             info="Open LLM Leaderboard v2 benchmark datasets"
         )
         metric_dropdown = gr.Dropdown(
-            choices=["Kappa_p (prob.)", "Kappa_p (det.)", "Error Consistency"],
+            choices=["CAPA", "CAPA (det.)", "Error Consistency"],
             label="Select Metric",
             info="Select a similarity metric to compute"
         )
@@ -158,7 +158,7 @@ with gr.Blocks(title="LLM Similarity Analyzer") as demo:
 - **Models**: Open LLM Leaderboard models \n
     - Every model evaluation is gated on Hugging Face and access has to be requested. \n
     - We requested access for the most popular models, but some may be missing. \n
-- **Metrics**: Kappa_p (probabilistic), Kappa_p (deterministic), Error Consistency""")
+- **Metrics**: CAPA (probabilistic), CAPA (deterministic), Error Consistency""")
 
 if __name__ == "__main__":
     demo.launch(ssr_mode=False)

src/dataloading.py

@@ -9,17 +9,19 @@ def get_leaderboard_models():
     api = HfApi()
     
     # List all datasets in the open-llm-leaderboard organization
-    datasets = api.list_datasets(author="open-llm-leaderboard")
+    dataset_list = api.list_datasets(author="open-llm-leaderboard")
     
     models = []
-    for dataset in datasets:
+    for dataset in dataset_list:
         if dataset.id.endswith("-details"):
             dataset_id = dataset.id
             try:
                 # Check if the dataset can be loaded
+                print(dataset_id)
                 check_gated = datasets.get_dataset_config_names(dataset_id)
+                print(check_gated)
                 # Format: "open-llm-leaderboard/<provider>__<model_name>-details"
-                model_part = dataset.id.split("/")[-1].replace("-details", "")
+                model_part = dataset_id.split("/")[-1].replace("-details", "")
                 if "__" in model_part:
                     provider, model = model_part.split("__", 1)
                     models.append(f"{provider}/{model}")

src/similarity.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from lmsim.metrics import Metrics, Kappa_p, EC
+from lmsim.metrics import Metrics, CAPA, EC
 
 from src.dataloading import load_run_data
 from src.utils import softmax, one_hot
@@ -32,9 +32,9 @@ def compute_similarity(metric: Metrics, outputs_a: list[np.array], outputs_b: li
 def compute_pairwise_similarities(metric_name: str, probs: list[list[np.array]], gts: list[list[int]]) -> np.array:
     # Select chosen metric
     if metric_name == "Kappa_p (prob.)":
-        metric = Kappa_p()
+        metric = CAPA()
     elif metric_name == "Kappa_p (det.)":
-        metric = Kappa_p(prob=False)
+        metric = CAPA(prob=False)
         # Convert probabilities to one-hot
         probs = [[one_hot(p) for p in model_probs] for model_probs in probs]
     elif metric_name == "Error Consistency":