Init commit with simple table

app.py

@@ -0,0 +1,110 @@
+import os
+import json
+import glob
+from collections import defaultdict
+import gradio as gr
+
+import glob
+
+ARC = "arc_challenge"
+HELLASWAG = "hellaswag"
+MMLU = "mmlu"
+TRUTHFULQA = "truthfulqa-mc"
+BENCHMARKS = [ARC, HELLASWAG, MMLU, TRUTHFULQA]
+
+METRICS = ["acc_norm", "acc_norm", "acc_norm", "mc2"]
+
+
+def collect_results():
+    performance_dict = defaultdict(dict)
+    pretrained_models = set()
+    for file in glob.glob('evals/*/*.json'):
+        with open(file, 'r') as f:
+            data = json.load(f)
+        if 'results' not in data:
+            continue
+        if 'config' not in data:
+            continue
+        results = data['results']
+        config = data['config']
+        if 'model_args' not in config:
+            continue
+
+        model_args = config['model_args'].split(',')
+        pretrained = [x for x in model_args if x.startswith('pretrained=')]
+        if len(pretrained) != 1:
+            continue
+        pretrained = pretrained[0].split('=')[1]
+        pretrained = pretrained.split('/')[-1]
+        pretrained_models.add(pretrained)
+
+        for lang_task, perfs in results.items():
+            if lang_task.startswith('arc_') and lang_task.endswith('_challenge'):
+                lang = lang_task.split('_')[1]
+                task = ARC
+            elif lang_task.startswith('hellaswag_'):
+                _, lang = lang_task.split('_')
+                task = HELLASWAG
+            elif lang_task.startswith('mmlu_'):
+                _, lang = lang_task.split('_')
+                task = MMLU
+            elif lang_task.startswith('truthfulqa_') and lang_task.endswith('_mc'):
+                lang = lang_task.split('_')[1]
+                task = TRUTHFULQA
+
+            if lang and task:
+                metric = METRICS[BENCHMARKS.index(task)]
+                p = round(perfs[metric] * 100, 1)
+                performance_dict[(pretrained, lang)][task] = p
+    return performance_dict, pretrained_models
+
+
+def get_leaderboard_df(performance_dict, pretrained_models):
+    df = list()
+    for (pretrained, lang), perfs in performance_dict.items():
+        arc_perf = perfs.get(ARC, 0.0)
+        hellaswag_perf = perfs.get(HELLASWAG, 0.0)
+        mmlu_perf = perfs.get(MMLU, 0.0)
+        truthfulqa_perf = perfs.get(TRUTHFULQA, 0.0)
+
+        if arc_perf * hellaswag_perf * mmlu_perf * truthfulqa_perf == 0:
+            continue
+        avg = round((arc_perf + hellaswag_perf + mmlu_perf + truthfulqa_perf) / 4, 1)
+        row = [pretrained, lang, avg, arc_perf, hellaswag_perf, mmlu_perf, truthfulqa_perf]
+        df.append(row)
+    return df
+
+
+MODEL_COL = "Model"
+LANG_COL = "Language"
+AVERAGE_COL = "Average"
+ARC_COL = "ARC (25-shot)"
+HELLASWAG_COL = "HellaSwag (10-shot)️"
+MMLU_COL = "MMLU (5-shot))️"
+TRUTHFULQA_COL = "TruthfulQA (0-shot)"
+
+COLS = [MODEL_COL, LANG_COL, AVERAGE_COL, ARC_COL, HELLASWAG_COL, MMLU_COL, TRUTHFULQA_COL]
+TYPES = ["str", "str", "number", "number", "number", "number", "number"]
+
+args = collect_results()
+leaderboard_df = get_leaderboard_df(*args)
+
+demo = gr.Blocks()
+with demo:
+    gr.HTML('Open Multilingual Large Language Model Evaluation Leaderboard')
+    gr.Markdown('INTRODUCTION TEXT', elem_classes="markdown-text")
+
+    with gr.Box():
+        search_bar = gr.Textbox(
+            placeholder="Search models...", show_label=False, elem_id="search-bar"
+        )
+
+        leaderboard_table = gr.components.Dataframe(
+            value=leaderboard_df,
+            headers=COLS,
+            datatype=TYPES,
+            max_rows=5,
+            elem_id="leaderboard-table",
+        )
+
+demo.launch()

evals/arc-challenge/arc_ar_challenge_bloom-1b7.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ar_challenge": {
+      "acc": 0.22818791946308725,
+      "acc_stderr": 0.02435139725761051,
+      "acc_norm": 0.2516778523489933,
+      "acc_norm_stderr": 0.025181904610615872
+    }
+  },
+  "versions": {
+    "arc_ar_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-1b7",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ar_challenge_bloom-560.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ar_challenge": {
+      "acc": 0.2550335570469799,
+      "acc_stderr": 0.025292327380712708,
+      "acc_norm": 0.2550335570469799,
+      "acc_norm_stderr": 0.025292327380712708
+    }
+  },
+  "versions": {
+    "arc_ar_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-560m",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ar_challenge_bloom-7b1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ar_challenge": {
+      "acc": 0.28187919463087246,
+      "acc_stderr": 0.026106703750007426,
+      "acc_norm": 0.3087248322147651,
+      "acc_norm_stderr": 0.026806063072940547
+    }
+  },
+  "versions": {
+    "arc_ar_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/bloom-7b1",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ar_challenge_gpt2-large.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ar_challenge": {
+      "acc": 0.20134228187919462,
+      "acc_stderr": 0.023268565767685306,
+      "acc_norm": 0.21476510067114093,
+      "acc_norm_stderr": 0.023828868848284352
+    }
+  },
+  "versions": {
+    "arc_ar_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-large",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ar_challenge_gpt2-medium.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ar_challenge": {
+      "acc": 0.19463087248322147,
+      "acc_stderr": 0.022973392306598162,
+      "acc_norm": 0.21140939597315436,
+      "acc_norm_stderr": 0.02369243605357901
+    }
+  },
+  "versions": {
+    "arc_ar_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-medium",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ar_challenge_gpt2.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ar_challenge": {
+      "acc": 0.20134228187919462,
+      "acc_stderr": 0.023268565767685313,
+      "acc_norm": 0.22483221476510068,
+      "acc_norm_stderr": 0.024224169829650755
+    }
+  },
+  "versions": {
+    "arc_ar_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ar_challenge_llama-7B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ar_challenge": {
+      "acc": 0.22483221476510068,
+      "acc_stderr": 0.02422416982965075,
+      "acc_norm": 0.24161073825503357,
+      "acc_norm_stderr": 0.024838535108028477
+    }
+  },
+  "versions": {
+    "arc_ar_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/llama-7B",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_bn_challenge_bloom-1b7.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_bn_challenge": {
+      "acc": 0.20945945945945946,
+      "acc_stderr": 0.023691963473475724,
+      "acc_norm": 0.2533783783783784,
+      "acc_norm_stderr": 0.025323518629100008
+    }
+  },
+  "versions": {
+    "arc_bn_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-1b7",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_bn_challenge_bloom-560.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_bn_challenge": {
+      "acc": 0.22972972972972974,
+      "acc_stderr": 0.024491712953916975,
+      "acc_norm": 0.24662162162162163,
+      "acc_norm_stderr": 0.025096383517594287
+    }
+  },
+  "versions": {
+    "arc_bn_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-560m",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_bn_challenge_bloom-7b1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_bn_challenge": {
+      "acc": 0.23986486486486486,
+      "acc_stderr": 0.02486094967084638,
+      "acc_norm": 0.28040540540540543,
+      "acc_norm_stderr": 0.026153277917823237
+    }
+  },
+  "versions": {
+    "arc_bn_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/bloom-7b1",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_bn_challenge_gpt2-large.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_bn_challenge": {
+      "acc": 0.2195945945945946,
+      "acc_stderr": 0.024102381106046785,
+      "acc_norm": 0.2668918918918919,
+      "acc_norm_stderr": 0.025753762926257924
+    }
+  },
+  "versions": {
+    "arc_bn_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-large",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_bn_challenge_gpt2-medium.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_bn_challenge": {
+      "acc": 0.20608108108108109,
+      "acc_stderr": 0.02355028295929425,
+      "acc_norm": 0.24662162162162163,
+      "acc_norm_stderr": 0.02509638351759427
+    }
+  },
+  "versions": {
+    "arc_bn_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-medium",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_bn_challenge_gpt2.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_bn_challenge": {
+      "acc": 0.22635135135135134,
+      "acc_stderr": 0.024364215012920555,
+      "acc_norm": 0.2668918918918919,
+      "acc_norm_stderr": 0.025753762926257917
+    }
+  },
+  "versions": {
+    "arc_bn_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_bn_challenge_llama-7B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_bn_challenge": {
+      "acc": 0.22635135135135134,
+      "acc_stderr": 0.024364215012920565,
+      "acc_norm": 0.26013513513513514,
+      "acc_norm_stderr": 0.02554257639364025
+    }
+  },
+  "versions": {
+    "arc_bn_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/llama-7B",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ca_challenge_bloom-1b7.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ca_challenge": {
+      "acc": 0.2356902356902357,
+      "acc_stderr": 0.02466946003490763,
+      "acc_norm": 0.27946127946127947,
+      "acc_norm_stderr": 0.026082164400369843
+    }
+  },
+  "versions": {
+    "arc_ca_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-1b7",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ca_challenge_bloom-560.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ca_challenge": {
+      "acc": 0.2053872053872054,
+      "acc_stderr": 0.02348110951859932,
+      "acc_norm": 0.23232323232323232,
+      "acc_norm_stderr": 0.02454650495612789
+    }
+  },
+  "versions": {
+    "arc_ca_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-560m",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ca_challenge_bloom-7b1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ca_challenge": {
+      "acc": 0.3164983164983165,
+      "acc_stderr": 0.02703395838420779,
+      "acc_norm": 0.3434343434343434,
+      "acc_norm_stderr": 0.0276003816062635
+    }
+  },
+  "versions": {
+    "arc_ca_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/bloom-7b1",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ca_challenge_gpt2-large.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ca_challenge": {
+      "acc": 0.20875420875420875,
+      "acc_stderr": 0.02362258775627148,
+      "acc_norm": 0.22895622895622897,
+      "acc_norm_stderr": 0.02442136264227106
+    }
+  },
+  "versions": {
+    "arc_ca_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-large",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ca_challenge_gpt2-medium.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ca_challenge": {
+      "acc": 0.20875420875420875,
+      "acc_stderr": 0.023622587756271473,
+      "acc_norm": 0.21212121212121213,
+      "acc_norm_stderr": 0.023761611918761673
+    }
+  },
+  "versions": {
+    "arc_ca_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-medium",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ca_challenge_gpt2.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ca_challenge": {
+      "acc": 0.21885521885521886,
+      "acc_stderr": 0.024032467624412215,
+      "acc_norm": 0.21885521885521886,
+      "acc_norm_stderr": 0.02403246762441221
+    }
+  },
+  "versions": {
+    "arc_ca_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_ca_challenge_llama-7B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_ca_challenge": {
+      "acc": 0.29292929292929293,
+      "acc_stderr": 0.026452514969665927,
+      "acc_norm": 0.29292929292929293,
+      "acc_norm_stderr": 0.02645251496966592
+    }
+  },
+  "versions": {
+    "arc_ca_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/llama-7B",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_da_challenge_bloom-1b7.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_da_challenge": {
+      "acc": 0.2255892255892256,
+      "acc_stderr": 0.02429399929295737,
+      "acc_norm": 0.26262626262626265,
+      "acc_norm_stderr": 0.02557802773320011
+    }
+  },
+  "versions": {
+    "arc_da_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-1b7",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_da_challenge_bloom-560.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_da_challenge": {
+      "acc": 0.25925925925925924,
+      "acc_stderr": 0.025471492792791667,
+      "acc_norm": 0.24579124579124578,
+      "acc_norm_stderr": 0.025025521384235284
+    }
+  },
+  "versions": {
+    "arc_da_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-560m",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_da_challenge_bloom-7b1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_da_challenge": {
+      "acc": 0.24242424242424243,
+      "acc_stderr": 0.02490893747050877,
+      "acc_norm": 0.24915824915824916,
+      "acc_norm_stderr": 0.025140041284626418
+    }
+  },
+  "versions": {
+    "arc_da_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/bloom-7b1",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_da_challenge_gpt2-large.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_da_challenge": {
+      "acc": 0.23232323232323232,
+      "acc_stderr": 0.02454650495612789,
+      "acc_norm": 0.24242424242424243,
+      "acc_norm_stderr": 0.024908937470508753
+    }
+  },
+  "versions": {
+    "arc_da_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-large",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_da_challenge_gpt2-medium.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_da_challenge": {
+      "acc": 0.24579124579124578,
+      "acc_stderr": 0.0250255213842353,
+      "acc_norm": 0.2727272727272727,
+      "acc_norm_stderr": 0.025886127156886297
+    }
+  },
+  "versions": {
+    "arc_da_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-medium",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_da_challenge_gpt2.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_da_challenge": {
+      "acc": 0.2222222222222222,
+      "acc_stderr": 0.02416437978893547,
+      "acc_norm": 0.23905723905723905,
+      "acc_norm_stderr": 0.024790260423468984
+    }
+  },
+  "versions": {
+    "arc_da_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_da_challenge_llama-7B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_da_challenge": {
+      "acc": 0.3063973063973064,
+      "acc_stderr": 0.026794891419479452,
+      "acc_norm": 0.3367003367003367,
+      "acc_norm_stderr": 0.02746823841289221
+    }
+  },
+  "versions": {
+    "arc_da_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/llama-7B",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_de_challenge_bloom-1b7.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_de_challenge": {
+      "acc": 0.24496644295302014,
+      "acc_stderr": 0.024955035980898946,
+      "acc_norm": 0.2953020134228188,
+      "acc_norm_stderr": 0.026470155629081085
+    }
+  },
+  "versions": {
+    "arc_de_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-1b7",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_de_challenge_bloom-560.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_de_challenge": {
+      "acc": 0.2348993288590604,
+      "acc_stderr": 0.024599255015999244,
+      "acc_norm": 0.28187919463087246,
+      "acc_norm_stderr": 0.026106703750007426
+    }
+  },
+  "versions": {
+    "arc_de_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-560m",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_de_challenge_bloom-7b1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_de_challenge": {
+      "acc": 0.2684563758389262,
+      "acc_stderr": 0.0257145395148175,
+      "acc_norm": 0.2684563758389262,
+      "acc_norm_stderr": 0.0257145395148175
+    }
+  },
+  "versions": {
+    "arc_de_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/bloom-7b1",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_de_challenge_gpt2-large.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_de_challenge": {
+      "acc": 0.23825503355704697,
+      "acc_stderr": 0.024719951493159625,
+      "acc_norm": 0.27181208053691275,
+      "acc_norm_stderr": 0.025815342279487567
+    }
+  },
+  "versions": {
+    "arc_de_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-large",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_de_challenge_gpt2-medium.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_de_challenge": {
+      "acc": 0.23825503355704697,
+      "acc_stderr": 0.024719951493159625,
+      "acc_norm": 0.28859060402684567,
+      "acc_norm_stderr": 0.026291942108676806
+    }
+  },
+  "versions": {
+    "arc_de_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-medium",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_de_challenge_gpt2.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_de_challenge": {
+      "acc": 0.22483221476510068,
+      "acc_stderr": 0.02422416982965075,
+      "acc_norm": 0.21140939597315436,
+      "acc_norm_stderr": 0.02369243605357901
+    }
+  },
+  "versions": {
+    "arc_de_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_de_challenge_llama-7B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_de_challenge": {
+      "acc": 0.2785234899328859,
+      "acc_stderr": 0.0260114035784859,
+      "acc_norm": 0.348993288590604,
+      "acc_norm_stderr": 0.027658144793750224
+    }
+  },
+  "versions": {
+    "arc_de_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/llama-7B",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_es_challenge_bloom-1b7.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_es_challenge": {
+      "acc": 0.2356902356902357,
+      "acc_stderr": 0.02466946003490763,
+      "acc_norm": 0.2895622895622896,
+      "acc_norm_stderr": 0.026362594432681956
+    }
+  },
+  "versions": {
+    "arc_es_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-1b7",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_es_challenge_bloom-560.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_es_challenge": {
+      "acc": 0.2255892255892256,
+      "acc_stderr": 0.024293999292957367,
+      "acc_norm": 0.2356902356902357,
+      "acc_norm_stderr": 0.02466946003490764
+    }
+  },
+  "versions": {
+    "arc_es_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-560m",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_es_challenge_bloom-7b1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_es_challenge": {
+      "acc": 0.3265993265993266,
+      "acc_stderr": 0.027258287015652305,
+      "acc_norm": 0.3602693602693603,
+      "acc_norm_stderr": 0.02790399493827167
+    }
+  },
+  "versions": {
+    "arc_es_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/bloom-7b1",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_es_challenge_gpt2-large.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_es_challenge": {
+      "acc": 0.2222222222222222,
+      "acc_stderr": 0.024164379788935483,
+      "acc_norm": 0.26262626262626265,
+      "acc_norm_stderr": 0.02557802773320012
+    }
+  },
+  "versions": {
+    "arc_es_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-large",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_es_challenge_gpt2-medium.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_es_challenge": {
+      "acc": 0.1919191919191919,
+      "acc_stderr": 0.022889733897083934,
+      "acc_norm": 0.25252525252525254,
+      "acc_norm_stderr": 0.02525252525252536
+    }
+  },
+  "versions": {
+    "arc_es_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-medium",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_es_challenge_gpt2.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_es_challenge": {
+      "acc": 0.19865319865319866,
+      "acc_stderr": 0.023190610381322127,
+      "acc_norm": 0.24579124579124578,
+      "acc_norm_stderr": 0.0250255213842353
+    }
+  },
+  "versions": {
+    "arc_es_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_es_challenge_llama-7B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_es_challenge": {
+      "acc": 0.3501683501683502,
+      "acc_stderr": 0.027726370308831506,
+      "acc_norm": 0.3602693602693603,
+      "acc_norm_stderr": 0.02790399493827167
+    }
+  },
+  "versions": {
+    "arc_es_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/llama-7B",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_eu_challenge_bloom-1b7.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_eu_challenge": {
+      "acc": 0.22377622377622378,
+      "acc_stderr": 0.02468755105337312,
+      "acc_norm": 0.2517482517482518,
+      "acc_norm_stderr": 0.02570896966075011
+    }
+  },
+  "versions": {
+    "arc_eu_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-1b7",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_eu_challenge_bloom-560.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_eu_challenge": {
+      "acc": 0.24475524475524477,
+      "acc_stderr": 0.02546756553847068,
+      "acc_norm": 0.19230769230769232,
+      "acc_norm_stderr": 0.023345268410264786
+    }
+  },
+  "versions": {
+    "arc_eu_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=bigscience/bloom-560m",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_eu_challenge_bloom-7b1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_eu_challenge": {
+      "acc": 0.23076923076923078,
+      "acc_stderr": 0.024957141712425013,
+      "acc_norm": 0.24125874125874125,
+      "acc_norm_stderr": 0.025343462496583764
+    }
+  },
+  "versions": {
+    "arc_eu_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/bloom-7b1",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_eu_challenge_gpt2-large.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_eu_challenge": {
+      "acc": 0.25874125874125875,
+      "acc_stderr": 0.02594151450124707,
+      "acc_norm": 0.24125874125874125,
+      "acc_norm_stderr": 0.025343462496583737
+    }
+  },
+  "versions": {
+    "arc_eu_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-large",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_eu_challenge_gpt2-medium.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_eu_challenge": {
+      "acc": 0.2762237762237762,
+      "acc_stderr": 0.026485626798716442,
+      "acc_norm": 0.25874125874125875,
+      "acc_norm_stderr": 0.025941514501247064
+    }
+  },
+  "versions": {
+    "arc_eu_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2-medium",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_eu_challenge_gpt2.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_eu_challenge": {
+      "acc": 0.2762237762237762,
+      "acc_stderr": 0.026485626798716456,
+      "acc_norm": 0.24825174825174826,
+      "acc_norm_stderr": 0.025589390464738234
+    }
+  },
+  "versions": {
+    "arc_eu_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=gpt2",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc-challenge/arc_eu_challenge_llama-7B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_eu_challenge": {
+      "acc": 0.26223776223776224,
+      "acc_stderr": 0.026054539173797044,
+      "acc_norm": 0.23426573426573427,
+      "acc_norm_stderr": 0.02508828621716978
+    }
+  },
+  "versions": {
+    "arc_eu_challenge": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/llama-7B",
+    "batch_size": "1",
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}