fix: Update win ratios to take ranks into account

app.py

@@ -232,52 +232,6 @@ DATASETS = [
 ]
 
 
-def update_colour_mapping(results_dfs: dict[Language, pd.DataFrame]) -> None:
-    """Get a mapping from model ids to RGB triplets.
-
-    Args:
-        results_dfs:
-            The results dataframes for each language.
-    """
-    global colour_mapping
-    global seed
-    seed += 1
-
-    gr.Info(f"Updating colour mapping...")
-
-    # Get distinct RGB values for all models
-    all_models = list(
-        {model_id for df in results_dfs.values() for model_id in df.index}
-    )
-    colour_mapping = dict()
-
-    for i in it.count():
-        min_colour_distance = MIN_COLOUR_DISTANCE_BETWEEN_MODELS - i
-        retries_left = 10 * len(all_models)
-        for model_id in all_models:
-            random.seed(hash(model_id) + i + seed)
-            r, g, b = 0, 0, 0
-            too_bright, similar_to_other_model = True, True
-            while (too_bright or similar_to_other_model) and retries_left > 0:
-                r, g, b = tuple(random.randint(0, 255) for _ in range(3))
-                too_bright = np.min([r, g, b]) > 200
-                similar_to_other_model = any(
-                    np.abs(
-                        np.array(colour) - np.array([r, g, b])
-                    ).sum() < min_colour_distance
-                    for colour in colour_mapping.values()
-                )
-                retries_left -= 1
-            colour_mapping[model_id] = (r, g, b)
-
-        if retries_left:
-            logger.info(
-                f"Successfully found a colour mapping with min colour distance "
-                f"{min_colour_distance}."
-            )
-            break
-
-
 def main() -> None:
     """Produce a radial plot."""
 
@@ -560,26 +514,61 @@ def produce_radial_plot(
         if all(task in df.columns for df in results_dfs_filtered.values())
     ]
 
+
+    logger.info("Computing win ratios...")
+    all_win_ratios: dict[Task, dict[Language, dict[str, float]]] = {
+        task: {
+            language: dict()
+            for language in languages
+        }
+        for task in tasks
+    }
+    for task in tasks:
+        for language in languages:
+            df = results_dfs_filtered[language][task].dropna()
+            model_ids_sorted: list[str] = (
+                df.map(np.mean).sort_values(ascending=False).index.tolist()
+            )
+            ranks = list()
+            rank = 0
+            best_scores = None
+            for model_id in model_ids_sorted:
+                if best_scores is None:
+                    best_scores = df.loc[model_id]
+                    rank = 1
+                else:
+                    scores = df.loc[model_id]
+                    worse_than_previous_models = stats.ttest_rel(
+                        a=best_scores, b=scores, alternative="greater"
+                    ).pvalue < 0.05
+                    if worse_than_previous_models:
+                        rank += 1
+                        best_scores = scores
+                ranks.append(rank)
+
+            for model_id, rank in zip(model_ids_sorted, ranks):
+                pct_models_with_higher_rank = np.mean(np.asarray(ranks) >= rank)
+                all_win_ratios[task][language][model_id] = pct_models_with_higher_rank
+    logger.info("Successfully computed win ratios.")
+
     # Add all the evaluation results for each model
     results: list[list[float]] = list()
     for model_id in model_ids:
         result_list = list()
         for task in tasks:
+
             win_ratios = list()
             scores = list()
             for language in languages:
                 if model_id not in results_dfs_filtered[language].index:
                     continue
+
                 score_list = results_dfs_filtered[language].loc[model_id][task]
-                win_ratio = 100 * np.mean([
-                    stats.ttest_rel(
-                        a=score_list, b=other_scores, alternative="greater"
-                    ).pvalue < 0.05
-                    for other_scores in results_dfs_filtered[language][task].dropna().drop(index=model_id)
-                ])
+
+                win_ratio = 100 * all_win_ratios[task][language][model_id]
                 win_ratios.append(win_ratio)
 
-                if all(score < 1 for score in score_list):
+                if np.mean(score_list) < 1:
                     score_list = [100 * score for score in score_list]
 
                 scores.append(np.mean(score_list))
@@ -645,6 +634,7 @@ def produce_radial_plot(
 
     return fig
 
+
 def fetch_results() -> dict[Language, pd.DataFrame]:
     """Fetch the results from the ScandEval benchmark.
 
@@ -674,6 +664,12 @@ def fetch_results() -> dict[Language, pd.DataFrame]:
         data_dict = defaultdict(dict)
         for record in records:
             model_name = record["model"]
+
+            # Manual fix for OpenAI models: Only keep the validation split results
+            if "gpt-3.5" in model_name or "gpt-4" in model_name:
+                if not record.get("validation_split", False):
+                    continue
+
             dataset_name = record["dataset"]
             if dataset_name in possible_dataset_names:
                 dataset = next(
@@ -702,5 +698,52 @@ def fetch_results() -> dict[Language, pd.DataFrame]:
 
     return results_dfs
 
+
+def update_colour_mapping(results_dfs: dict[Language, pd.DataFrame]) -> None:
+    """Get a mapping from model ids to RGB triplets.
+
+    Args:
+        results_dfs:
+            The results dataframes for each language.
+    """
+    global colour_mapping
+    global seed
+    seed += 1
+
+    gr.Info(f"Updating colour mapping...")
+
+    # Get distinct RGB values for all models
+    all_models = list(
+        {model_id for df in results_dfs.values() for model_id in df.index}
+    )
+    colour_mapping = dict()
+
+    for i in it.count():
+        min_colour_distance = MIN_COLOUR_DISTANCE_BETWEEN_MODELS - i
+        retries_left = 10 * len(all_models)
+        for model_id in all_models:
+            random.seed(hash(model_id) + i + seed)
+            r, g, b = 0, 0, 0
+            too_bright, similar_to_other_model = True, True
+            while (too_bright or similar_to_other_model) and retries_left > 0:
+                r, g, b = tuple(random.randint(0, 255) for _ in range(3))
+                too_bright = np.min([r, g, b]) > 200
+                similar_to_other_model = any(
+                    np.abs(
+                        np.array(colour) - np.array([r, g, b])
+                    ).sum() < min_colour_distance
+                    for colour in colour_mapping.values()
+                )
+                retries_left -= 1
+            colour_mapping[model_id] = (r, g, b)
+
+        if retries_left:
+            logger.info(
+                f"Successfully found a colour mapping with min colour distance "
+                f"{min_colour_distance}."
+            )
+            break
+
+
 if __name__ == "__main__":
     main()