always lower case shortname

app.py

@@ -1,5 +1,7 @@
 import json
 from collections import defaultdict
+from dataclasses import dataclass, field
+from functools import cached_property
 from pathlib import Path
 
 import numpy as np
@@ -18,6 +20,51 @@ BENCHMARKS = [ARC, HELLASWAG, MMLU, TRUTHFULQA]
 
 METRICS = ["acc_norm", "acc_norm", "acc_norm", "mc2"]
 
+MODEL_COL = "Model"
+AVERAGE_COL = "Average"
+ARC_COL = "ARC (25-shot)"
+HELLASWAG_COL = "HellaSwag (10-shot)️"
+MMLU_COL = "MMLU (5-shot)"
+TRUTHFULQA_COL = "TruthfulQA (0-shot)"
+TRAIN_TYPE_COL = "Training type"
+TRAIN_TYPE_COL = "Training type"
+NUM_PARAMETERS = "Num. parameters"
+
+
+@dataclass
+class Result:
+    train_type: str
+    num_parameters: int
+    arc: float = field(default=0.)
+    hellaswag: float = field(default=0.)
+    mmlu: float = field(default=0.)
+    truthfulqa: float = field(default=0.)
+
+    @cached_property
+    def num_parameters_kmb(self) -> str:
+        return convert_number_to_kmb(self.num_parameters)
+
+    @cached_property
+    def average(self) -> float:
+        return self.arc + self.hellaswag + self.mmlu + self.truthfulqa / 4
+
+
+def convert_number_to_kmb(number: int) -> str:
+    """
+    Converts a number to a string with K, M or B suffix
+    :param number: the number to convert
+    :return: a string with the number and a suffix, e.g. "7B", rounded to one decimal
+    """
+    if number >= 1_000_000_000:
+        return f"{round(number / 1_000_000_000, 1)}B"
+    elif number >= 1_000_000:
+        return f"{round(number / 1_000_000, 1)}M"
+    elif number >= 1_000:
+        return f"{round(number / 1_000, 1)}K"
+    else:
+        return str(number)
+
+
 
 def collect_results() -> dict[tuple[str, str], dict[str, float]]:
     """
@@ -104,6 +151,8 @@ HELLASWAG_COL = "HellaSwag (10-shot)️"
 MMLU_COL = "MMLU (5-shot)"
 TRUTHFULQA_COL = "TruthfulQA (0-shot)"
 TRAIN_TYPE_COL = "Training type"
+TRAIN_TYPE_COL = "Training type"
+NUM_PARAMETERS = "Num. parameters"
 
 COLS = [MODEL_COL, TRAIN_TYPE_COL, AVERAGE_COL, ARC_COL, HELLASWAG_COL, MMLU_COL, TRUTHFULQA_COL]
 TYPES = ["str", "number", "number", "number", "number", "number"]

evals/models.json

@@ -1,144 +0,0 @@
-{
-    "Llama-2-13b-chat-dutch": {
-        "compute_dtype": "bfloat16",
-        "model_name": "BramVanroy/Llama-2-13b-chat-dutch",
-        "num_parameters": 13015864320,
-        "quantization": "8-bit"
-    },
-    "Llama-2-13b-chat-hf": {
-        "compute_dtype": "bfloat16",
-        "model_name": "meta-llama/Llama-2-13b-chat-hf",
-        "num_parameters": 13015864320,
-        "quantization": "8-bit"
-    },
-    "Llama-2-13b-hf": {
-        "compute_dtype": "bfloat16",
-        "model_name": "meta-llama/Llama-2-13b-hf",
-        "num_parameters": 13015864320,
-        "quantization": "8-bit"
-    },
-    "Llama-2-7b-chat-hf": {
-        "compute_dtype": "bfloat16",
-        "model_name": "meta-llama/Llama-2-7b-chat-hf",
-        "num_parameters": 6738415616,
-        "quantization": "8-bit"
-    },
-    "Llama-2-7b-hf": {
-        "compute_dtype": "bfloat16",
-        "model_name": "meta-llama/Llama-2-7b-hf",
-        "num_parameters": 6738415616,
-        "quantization": "8-bit"
-    },
-    "Mistral-7B-v0.1": {
-        "compute_dtype": "bfloat16",
-        "model_name": "mistralai/Mistral-7B-v0.1",
-        "num_parameters": 7241732096,
-        "quantization": "8-bit"
-    },
-    "Orca-2-13b": {
-        "compute_dtype": "bfloat16",
-        "model_name": "microsoft/Orca-2-13b",
-        "num_parameters": 13015895040,
-        "quantization": "8-bit"
-    },
-    "Orca-2-7b": {
-        "compute_dtype": "bfloat16",
-        "model_name": "microsoft/Orca-2-7b",
-        "num_parameters": 6738440192,
-        "quantization": "8-bit"
-    },
-    "bloom-7b1": {
-        "args": "pretrained=bigscience/bloom-7b1",
-        "model_name": "pretrained=bigscience/bloom-7b1"
-    },
-    "gpt-neo-1.3B-dutch": {
-        "compute_dtype": "bfloat16",
-        "model_name": "yhavinga/gpt-neo-1.3B-dutch",
-        "num_parameters": 1315575808,
-        "quantization": "8-bit"
-    },
-    "gpt-neo-1.3b-dutch": {
-        "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True",
-        "model_name": "yhavinga/gpt-neo-1.3B-dutch"
-    },
-    "gpt-neo-125M-dutch": {
-        "compute_dtype": "bfloat16",
-        "model_name": "yhavinga/gpt-neo-125M-dutch",
-        "num_parameters": 125198592,
-        "quantization": "8-bit"
-    },
-    "gpt-neo-125m-dutch": {
-        "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True",
-        "model_name": "yhavinga/gpt-neo-125M-dutch"
-    },
-    "gpt2-large-dutch": {
-        "compute_dtype": "bfloat16",
-        "model_name": "yhavinga/gpt2-large-dutch",
-        "num_parameters": 774030080,
-        "quantization": "8-bit"
-    },
-    "gpt2-medium-dutch": {
-        "compute_dtype": "bfloat16",
-        "model_name": "yhavinga/gpt2-medium-dutch",
-        "num_parameters": 354823168,
-        "quantization": "8-bit"
-    },
-    "llama-2-13b-chat-dutch": {
-        "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True",
-        "model_name": "BramVanroy/Llama-2-13b-chat-dutch"
-    },
-    "llama-2-13b-chat-hf": {
-        "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True",
-        "model_name": "meta-llama/Llama-2-13b-chat-hf"
-    },
-    "llama-2-13b-hf": {
-        "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True",
-        "model_name": "meta-llama/Llama-2-13b-hf"
-    },
-    "llama-2-7b-chat-hf": {
-        "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True",
-        "model_name": "meta-llama/Llama-2-7b-chat-hf"
-    },
-    "llama-2-7b-hf": {
-        "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True",
-        "model_name": "meta-llama/Llama-2-7b-hf"
-    },
-    "llama-7b": {
-        "args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/llama-7B",
-        "model_name": "pretrained=/sensei-fs/users/daclai/uoChatGPT/llama-7B"
-    },
-    "llama2-13b-ft-mc4": {
-        "compute_dtype": "bfloat16",
-        "model_name": "BramVanroy/llama2-13b-ft-mc4_nl_cleaned_tiny",
-        "num_parameters": 13015864320,
-        "quantization": "8-bit"
-    },
-    "llama2-13b-ft-mc4_nl_cleaned_tiny": {
-        "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True",
-        "model_name": "BramVanroy/llama2-13b-ft-mc4_nl_cleaned_tiny"
-    },
-    "mistral-7b-v0.1": {
-        "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True",
-        "model_name": "mistralai/Mistral-7B-v0.1"
-    },
-    "neural-chat-7b-v3-1": {
-        "compute_dtype": "bfloat16",
-        "model_name": "Intel/neural-chat-7b-v3-1",
-        "num_parameters": 7241732096,
-        "quantization": "8-bit"
-    },
-    "orca-2-13b": {
-        "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True",
-        "model_name": "microsoft/Orca-2-13b"
-    },
-    "orca-2-7b": {
-        "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True",
-        "model_name": "microsoft/Orca-2-7b"
-    },
-    "zephyr-7b-beta": {
-        "compute_dtype": "bfloat16",
-        "model_name": "HuggingFaceH4/zephyr-7b-beta",
-        "num_parameters": 7241732096,
-        "quantization": "8-bit"
-    }
-}

generate_overview_json.py

@@ -16,7 +16,7 @@ def main():
     for pfin in evals_dir.rglob("*.json"):
         if pfin.stem == "models":
             continue
-        short_name = pfin.stem.split("_")[2]
+        short_name = pfin.stem.split("_")[2].lower()
 
         data = json.loads(pfin.read_text(encoding="utf-8"))
         if "config" not in data: