new results with random

app.py

@@ -1,9 +1,11 @@
-from functools import partial
+
 import json
 
 import gradio as gr
 import pandas as pd
 
+from statistics import median
+
 
 print("Loading datasets...")
 
@@ -67,13 +69,13 @@ def get_data_cross_mmlu_overall(eval_mode='zero_shot', fillna=True, rank=True):
        
         try:
             overall_acc = [results['overall_acc'] for results in results_list]
-            overall_acc = sum(overall_acc) / len(overall_acc)
+            overall_acc = median(overall_acc)
 
             consistency_score_3 = [results['consistency_score_3'] for results in results_list]
-            consistency_score_3 = sum(consistency_score_3) / len(consistency_score_3)
+            consistency_score_3 = median(consistency_score_3)
 
             AC3_3 = [results['AC3_3'] for results in results_list]
-            AC3_3 = sum(AC3_3) / len(AC3_3)
+            AC3_3 = median(AC3_3)
 
         except:
             print(results_list)
@@ -126,21 +128,21 @@ def get_data_cross_mmlu_language(eval_mode='zero_shot', fillna=True, rank=True):
 
        
         try:
-            English = [results['language_acc']['English'] for results in results_list]
+            English    = [results['language_acc']['English'] for results in results_list]
             Vietnamese = [results['language_acc']['Vietnamese'] for results in results_list]
-            Chinese = [results['language_acc']['Chinese'] for results in results_list]
+            Chinese    = [results['language_acc']['Chinese'] for results in results_list]
             Indonesian = [results['language_acc']['Indonesian'] for results in results_list]
-            Filipino = [results['language_acc']['Filipino'] for results in results_list]
-            Spanish = [results['language_acc']['Spanish'] for results in results_list]
-            Malay = [results['language_acc']['Malay'] for results in results_list]
+            Filipino   = [results['language_acc']['Filipino'] for results in results_list]
+            Spanish    = [results['language_acc']['Spanish'] for results in results_list]
+            Malay      = [results['language_acc']['Malay'] for results in results_list]
 
-            English = sum(English) / len(English)
-            Vietnamese = sum(Vietnamese) / len(Vietnamese)
-            Chinese = sum(Chinese) / len(Chinese)
-            Indonesian = sum(Indonesian) / len(Indonesian)
-            Filipino = sum(Filipino) / len(Filipino)
-            Spanish = sum(Spanish) / len(Spanish)
-            Malay = sum(Malay) / len(Malay)
+            English    = median(English)
+            Vietnamese = median(Vietnamese) 
+            Chinese    = median(Chinese)
+            Indonesian = median(Indonesian)
+            Filipino   = median(Filipino)
+            Spanish    = median(Spanish)
+            Malay      = median(Malay)
 
 
         except:
@@ -208,13 +210,13 @@ def get_data_cross_logiqa_overall(eval_mode='zero_shot', fillna=True, rank=True)
        
         try:
             overall_acc = [results['overall_acc'] for results in results_list]
-            overall_acc = sum(overall_acc) / len(overall_acc)
+            overall_acc = median(overall_acc)
 
             consistency_score_3 = [results['consistency_score_3'] for results in results_list]
-            consistency_score_3 = sum(consistency_score_3) / len(consistency_score_3)
+            consistency_score_3 = median(consistency_score_3)
 
             AC3_3 = [results['AC3_3'] for results in results_list]
-            AC3_3 = sum(AC3_3) / len(AC3_3)
+            AC3_3 = median(AC3_3)
 
         except:
             print(results_list)
@@ -267,21 +269,21 @@ def get_data_cross_logiqa_language(eval_mode='zero_shot', fillna=True, rank=True
 
        
         try:
-            English = [results['language_acc']['English'] for results in results_list]
+            English    = [results['language_acc']['English'] for results in results_list]
             Vietnamese = [results['language_acc']['Vietnamese'] for results in results_list]
-            Chinese = [results['language_acc']['Chinese'] for results in results_list]
+            Chinese    = [results['language_acc']['Chinese'] for results in results_list]
             Indonesian = [results['language_acc']['Indonesian'] for results in results_list]
-            Filipino = [results['language_acc']['Filipino'] for results in results_list]
-            Spanish = [results['language_acc']['Spanish'] for results in results_list]
-            Malay = [results['language_acc']['Malay'] for results in results_list]
+            Filipino   = [results['language_acc']['Filipino'] for results in results_list]
+            Spanish    = [results['language_acc']['Spanish'] for results in results_list]
+            Malay      = [results['language_acc']['Malay'] for results in results_list]
 
-            English = sum(English) / len(English)
-            Vietnamese = sum(Vietnamese) / len(Vietnamese)
-            Chinese = sum(Chinese) / len(Chinese)
-            Indonesian = sum(Indonesian) / len(Indonesian)
-            Filipino = sum(Filipino) / len(Filipino)
-            Spanish = sum(Spanish) / len(Spanish)
-            Malay = sum(Malay) / len(Malay)
+            English    = median(English)
+            Vietnamese = median(Vietnamese) 
+            Chinese    = median(Chinese)
+            Indonesian = median(Indonesian)
+            Filipino   = median(Filipino)
+            Spanish    = median(Spanish)
+            Malay      = median(Malay)
 
 
         except:
@@ -346,14 +348,12 @@ def get_data_sg_eval(eval_mode='zero_shot', fillna=True, rank=True):
 
        
         try:
-            accuracy = [results['accuracy'] for results in results_list]
-            accuracy = sum(accuracy) / len(accuracy)
+            accuracy = median([results['accuracy'] for results in results_list])
 
         except:
             print(results_list)
             accuracy = -1
 
-
         res = {
             "Model Size (Params)": MODEL_TO_SIZE.get(model, ""),
             "Model": make_clickable_model(model, link=ALL_RESULTS[model]["model_link"]),
@@ -401,8 +401,7 @@ def get_data_us_eval(eval_mode='zero_shot', fillna=True, rank=True):
 
        
         try:
-            accuracy = [results['accuracy'] for results in results_list]
-            accuracy = sum(accuracy) / len(accuracy)
+            accuracy = median([results['accuracy'] for results in results_list])
 
         except:
             print(results_list)
@@ -456,8 +455,7 @@ def get_data_cn_eval(eval_mode='zero_shot', fillna=True, rank=True):
 
        
         try:
-            accuracy = [results['accuracy'] for results in results_list]
-            accuracy = sum(accuracy) / len(accuracy)
+            accuracy = median([results['accuracy'] for results in results_list])
 
         except:
             print(results_list)
@@ -511,8 +509,7 @@ def get_data_ph_eval(eval_mode='zero_shot', fillna=True, rank=True):
 
        
         try:
-            accuracy = [results['accuracy'] for results in results_list]
-            accuracy = sum(accuracy) / len(accuracy)
+            accuracy = median([results['accuracy'] for results in results_list])
 
         except:
             print(results_list)
@@ -789,8 +786,8 @@ with block:
                 with gr.TabItem("Overall"):
                     with gr.Row():
                         gr.components.Dataframe(
-                            PH_EVAL_ZERO_SHOT,
-                            datatype=["number", "markdown"] + ["number"] * len(PH_EVAL_ZERO_SHOT.columns),
+                            PH_EVAL_FIVE_SHOT,
+                            datatype=["number", "markdown"] + ["number"] * len(PH_EVAL_FIVE_SHOT.columns),
                             type="pandas",
                         )