eq bench

app.py

@@ -1,3 +1,4 @@
+import json
 import re
 
 import gradio as gr
@@ -25,12 +26,17 @@ with demo:
         header = [h.strip() for h in header]
         for i, line in enumerate(f):
             leaderboard_df.append(line.strip().split(",", 13))
+
+    metadata = json.load(open('metadata.json'))
+    for k, v in list(metadata.items()):
+        metadata[k.split(",")[0]] = v
+
     # create dataframe from list and header
     leaderboard_df = pd.DataFrame(leaderboard_df, columns=header)
     # filter column with value eq-bench_v2_pl
     print(header)
     leaderboard_df = leaderboard_df[(leaderboard_df["Benchmark Version"] == "eq-bench_v2_pl") | (
-                leaderboard_df["Benchmark Version"] == 'eq-bench_pl')]
+            leaderboard_df["Benchmark Version"] == 'eq-bench_pl')]
     # fix: ValueError: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().
 
     # leave only defined columns
@@ -48,6 +54,18 @@ with demo:
     leaderboard_df["Num Questions Parseable"] = leaderboard_df[["Num Questions Parseable", "Error"]].apply(
         lambda x: parse_parseable(x), axis=1)
 
+
+    def get_params(model_name):
+        if model_name in metadata:
+            return metadata[model_name]
+        return numpy.nan
+
+
+    leaderboard_df["Params"] = leaderboard_df["Model Path"].apply(lambda x: get_params(x))
+
+    # move column order
+    leaderboard_df = leaderboard_df[["Model Path", "Params", "Benchmark Score", "Num Questions Parseable", 'Error']]
+
     leaderboard_df["Model Path"] = leaderboard_df["Model Path"].apply(lambda x: make_clickable_model(x))
 
     # change value of column to nan
@@ -64,21 +82,24 @@ with demo:
     leaderboard_df = leaderboard_df.sort_values(by=["Benchmark Score", "Num Questions Parseable"],
                                                 ascending=[False, False])
 
-    #rename column
+    # rename column
     leaderboard_df = leaderboard_df.rename(columns={"Model Path": "Model"})
 
     leaderboard_df_styled = leaderboard_df.style.background_gradient(cmap="RdYlGn")
+    leaderboard_df_styled = leaderboard_df_styled.background_gradient(cmap="RdYlGn_r", subset=['Params'])
+
     rounding = {}
     # for col in ["Benchmark Score", "Num Questions Parseable"]:
 
     rounding["Benchmark Score"] = "{:.2f}"
     rounding["Num Questions Parseable"] = "{:.0f}"
+    rounding["Params"] = "{:.0f}"
     leaderboard_df_styled = leaderboard_df_styled.format(rounding)
 
     leaderboard_table = gr.components.Dataframe(
         value=leaderboard_df_styled,
         # headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-        datatype=['markdown', 'number', 'number', 'str'],
+        datatype=['markdown', 'number', 'number', 'number', 'str'],
         elem_id="leaderboard-table",
         interactive=False,
         visible=True,

metadata.json

@@ -0,0 +1,316 @@
+{
+  "Azurro/APT3-1B-Base": 1,
+  "HuggingFaceH4/zephyr-7b-alpha": 7,
+  "Voicelab/trurl-2-13b-academic": 13,
+  "HuggingFaceH4/zephyr-7b-beta": 7,
+  "Voicelab/trurl-2-7b": 7,
+  "mistralai/Mistral-7B-v0.1": 7,
+  "mistralai/Mistral-7B-v0.1,peft=lora/output/mistral-7b-v0.1-lora-pl/checkpoint-400/adapter_model": 7,
+  "mistralai/Mistral-7B-v0.1,peft=lora/output/mistral-7b-v0.1-lora-pl/checkpoint-200/adapter_model": 7,
+  "mistralai/Mistral-7B-v0.1,load_in_8bit=True": 7,
+  "Nondzu/zephyr-speakleash-007-pl-8192-32-16-0.05": 7,
+  "openchat/openchat-3.5-0106": 7,
+  "mistralai/Mistral-7B-v0.1,peft=lora/output/mistral-7b-v0.1-lora-pl/checkpoint-2000/adapter_model": 7,
+  "mistralai/Mistral-7B-v0.1,peft=lora/output/mistral-7b-v0.1-lora-pl/checkpoint-2200/adapter_model": 7,
+  "mistralai/Mistral-7B-Instruct-v0.1": 7,
+  "APT3-1B-Instruct-e1": 1,
+  "mistralai/Mistral-7B-v0.1,peft=lora/output/mistral-7b-v0.1-lora-pl/checkpoint-800/adapter_model": 7,
+  "mistralai/Mistral-7B-v0.1,peft=lora/output/mistral-7b-v0.1-lora-pl/checkpoint-600/adapter_model": 7,
+  "APT3-1B-Instruct-e2": 1,
+  "mistralai/Mistral-7B-v0.1,load_in_4bit=True": 7,
+  "speakleash/3-5B_high_base/epoch_2_hf": 3.5,
+  "speakleash/3-5B_high_base/epoch_1_hf": 3.5,
+  "speakleash/3-5B_high_base/epoch_0_hf": 3.5,
+  "speakleash/7B_high_base/epoch_1_hf": 7,
+  "speakleash/7B_high_base/epoch_0_hf": 7,
+  "Nondzu/zephyr-speakleash-010-pl-3072-32-16-0.01": 7,
+  "google/mt5-xl": 3.7,
+  "speakleash/7B_high_sft/epoch_2_base/epoch_2_hf": 7,
+  "OPI-PG/Qra-1b": 1,
+  "OPI-PG/Qra-13b": 13,
+  "OPI-PG/Qra-7b": 7,
+  "teknium/OpenHermes-2.5-Mistral-7B": 7,
+  "openchat/openchat-3.5-1210": 7,
+  "speakleash/apt3-1B_base/apt3-1B-sequential_hf": 1,
+  "speakleash/apt3-1B_base/apt3-1B-shuffled_hf": 1,
+  "speakleash/1B_high_base/like_apt3-1B_hf": 1,
+  "speakleash/1B_high_base/epoch_3_hf": 1,
+  "speakleash/7B_high_sft/epoch_1_base/epoch_2_hf": 7,
+  "speakleash/7B_high_sft/epoch_1_base/epoch_1_hf": 7,
+  "speakleash/7B_high_sft/epoch_0_base/epoch_0_hf": 7,
+  "speakleash/7B_high_sft/epoch_2_base/epoch_1_hf": 7,
+  "speakleash/3-5B_high_sft/epoch_3_base/epoch_2_hf": 3.5,
+  "allegro/plt5-large": 0.82,
+  "internlm/internlm2-7b": 7,
+  "sdadas/polish-gpt2-xl": 1.67,
+  "speakleash/1B_4k_high_sft/epoch_3_base/epoch_1_hf": 1,
+  "speakleash/mistral-PL_7B/epoch_0_hf": 7,
+  "speakleash/1B_high_sft/epoch_3_base/epoch_1_hf": 1,
+  "speakleash/polish-mistral-7B/epoch_0_hf": 7,
+  "speakleash/3-5B_high_sft/epoch_0_base/epoch_2_hf": 3.5,
+  "speakleash/3-5B_high_sft/epoch_0_base/epoch_1_hf": 3.5,
+  "speakleash/3-5B_high_sft/epoch_0_base/epoch_0_hf": 3.5,
+  "speakleash/7B_high_base/epoch_2_hf": 7,
+  "speakleash/10B-4k_high_sft/epoch_3_base/epoch_1_hf": 10,
+  "speakleash/3-5B_high_base/epoch_3_hf": 3.5,
+  "microsoft/phi-2": 2.7,
+  "RWKV/HF_v5-Eagle-7B": 7,
+  "mistralai/Mistral-7B-Instruct-v0.2": 7,
+  "speakleash/llama-apt3-7B/only-spi-e0_hf": 7,
+  "speakleash/llama-apt3-7B/spkl-only_sft/e4_hf": 7,
+  "speakleash/llama-apt3-7B/spkl-only_sft/e5_hf": 7,
+  "speakleash/llama-apt3-7B/spkl-only_sft/e3_hf": 7,
+  "speakleash/llama-apt3-7B/spkl-only_sft/e2_hf": 7,
+  "meta-llama/Llama-2-7b-hf": 7,
+  "meta-llama/Llama-2-7b-chat-hf": 7,
+  "internlm/internlm2-chat-7b": 7,
+  "internlm/internlm2-base-7b": 7,
+  "internlm/internlm2-1_8b": 1.8,
+  "internlm/internlm2-chat-1_8b": 1.8,
+  "speakleash/mistral-apt3-7B/only-spi_sft/e0_hf": 7,
+  "speakleash/mistral-apt3-7B/only-spi-e0_hf": 7,
+  "speakleash/mistral-apt3-7B/apt3-e0_hf": 7,
+  "speakleash/mistral-apt3-7B/spi-e0_hf": 7,
+  "speakleash/mistral-apt3-7B/spkl_sft_v2/e4_hf": 7,
+  "speakleash/mistral-apt3-7B/spkl_sft_v2/e5_hf": 7,
+  "speakleash/mistral-apt3-7B/spkl_sft_v2/e3_hf": 7,
+  "speakleash/mistral-apt3-7B/spkl_sft_v2/e2_hf": 7,
+  "speakleash/mistral-apt3-7B/only-spi_sft_v2/e4_bb62a5b8": 7,
+  "speakleash/mistral-apt3-7B/only-spi_sft_v2/e6_6b0aa8d6": 7,
+  "speakleash/mistral-apt3-7B/only-spi_sft_v2/e3_f8b5e568": 7,
+  "speakleash/mistral-apt3-7B/only-spi_sft_v2/e2_3b7fc53e": 7,
+  "speakleash/mistral-apt3-7B/only-spi_sft_v2/e5_f75cbc76": 7,
+  "speakleash/mistral-apt3-7B/only-spi_sft_v2/e7_642f3822": 7,
+  "speakleash/mistral-apt3-7B/spkl_sft/e3_17ef3119": 7,
+  "speakleash/mistral-apt3-7B/spkl_sft/e2_7dc8df86": 7,
+  "google/gemma-7b": 7,
+  "google/gemma-7b-it": 7,
+  "SOTA FT HerBERT (large)": 1,
+  "Baseline (majority class)": 0,
+  "SOTA FT Polish RoBERTa": 1,
+  "SOTA FT ULMFiT-SP-PL": 0.1,
+  "speakleash/llama-apt3-13B/spkl-plus/e0_caa5ad79": 13,
+  "speakleash/llama-apt3-13B/spkl-only/e0_cc0931c5": 13,
+  "eryk-mazus/polka-1.1b": 1.1,
+  "berkeley-nest/Starling-LM-7B-alpha": 7,
+  "Remek/OpenChat3.5-0106-Spichlerz-Inst-001": 7,
+  "speakleash/mistral_7B-v2/spkl-all-e2_5bd6027d": 7,
+  "speakleash/mistral_7B-v2/spkl-all-e0_8cf0987d": 7,
+  "speakleash/mistral_7B-v2/spkl-only-e0_ef715d74": 7,
+  "speakleash/mistral_7B-v2/spkl-only-e1_333887a5": 7,
+  "speakleash/mistral_7B-v2/spkl-all-e1_0b514ce9": 7,
+  "speakleash/mistral_7B-v2/spkl-only-e2_5dac700d": 7,
+  "speakleash/llama-apt3-13B/spkl-only_e0_sft/ext_e3_23b6bc9b": 13,
+  "speakleash/llama-apt3-13B/spkl-only_e0_sft/spkl_e4_e3a666b1": 13,
+  "speakleash/llama-apt3-13B/spkl-only_e0_sft/spkl_e3_45ef6b63": 13,
+  "speakleash/llama-apt3-13B/spkl-only_e0_sft/spkl_e5_bf95416b": 13,
+  "speakleash/llama-apt3-13B/spkl-only_e0_sft/ext_e2_f7606252": 13,
+  "speakleash/llama-apt3-13B/spkl-only_e0_sft/spkl_e2_898ae6c6": 13,
+  "speakleash/apt4-1B/spkl-only-e3_756856c4": 1,
+  "speakleash/apt4-1B/spkl-all-e0_7f6a991e": 1,
+  "speakleash/apt4-1B/spkl-only-e2_969e76b4": 1,
+  "speakleash/apt4-1B/spkl-all-e2_bfb44ded": 1,
+  "speakleash/apt4-1B/spkl-all-e3_063753f9": 1,
+  "speakleash/apt4-1B/spkl-all-e1_74a293c8": 1,
+  "speakleash/apt4-1B/spkl-only-e0_b9c8bb39": 1,
+  "speakleash/apt4-1B/spkl-only-e1_fea4b41b": 1,
+  "upstage/SOLAR-10.7B-Instruct-v1.0": 10.7,
+  "upstage/SOLAR-10.7B-v1.0": 10.7,
+  "speakleash/mistral_7B-v2/spkl-all_sft/e1_base/spkl-all-e1_9aee511a": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft/e1_base/spkl-all-e0_dd9d2777": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft/e1_base/spkl-only-e1_d0ac34b7": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft/e1_base/spkl-only-e0_9eea5944": 7,
+  "Remek/Kruk-7B-SP-001": 7,
+  "TinyLlama/TinyLlama-1.1B-Chat-v1.0": 1.1,
+  "internlm/internlm2-chat-7b-sft": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft/e1_base/spkl-all-e3_72a6c52a": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft/e1_base/spkl-only-e3_08a0fd89": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft/e1_base/spkl-all-e2_0a1a62c0": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft/e1_base/spkl-only-e2_a7c66ac5": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_2e5-e0_116fa2bc": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_7e6-e0_8544bbd3": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_2e5-e1_013bd434": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only-e1_87bfffac": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only-e2_939d897f": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only-e0_2a5be0dc": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5/spkl-only-e1_0303962d": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5/spkl-only-e0_f4aaf490": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e0_base_2e5/spkl-only-e0_009b090e": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e0_base_2e5/spkl-only-e1_91aae327": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_7e6w-e1_14d52992": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_7e6w-e2_72422a32": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only-e2_dcb87efc": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_7e6-e2_04382c38": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_7e6-e3_860889b1": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_7e6w-e3_78cf3243": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_9e7-e0_27275908": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only-e0_d31a18b7": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_7e6-e0_c26126c8": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only-e3_a5833b75": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_7e6w-e0_6c834bf7": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_7e6-e1_87b7c12f": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_9e7-e2_5ce06dd2": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_9e7-e1_561ac4bb": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only-e1_392d55d9": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft_v2/e1_base/spkl-all_2e6-e2_db0cd739": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft_v2/e1_base/spkl-all_2e6-e3_4960543c": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft_v2/e1_base/spkl-all_2e6-e0_1b65c3ac": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft_v2/e1_base/spkl-all_2e6-e1_70c70cc6": 7,
+  "speakleash/mistral-apt3-7B-v2/spkl-only_sft/e1_base/spkl-only-e2_3a071212": 7,
+  "speakleash/mistral-apt3-7B-v2/spkl-only_sft/e1_base/spkl-only-e0_6dc2e217": 7,
+  "speakleash/mistral-apt3-7B-v2/spkl-only_sft/e1_base/spkl-only-e1_46610eb1": 7,
+  "speakleash/mistral-apt3-7B-v2/spkl-only_sft-weighted/e1_base/spkl-only-e0_e79dcb9f": 7,
+  "speakleash/mistral-apt3-7B-v2/spkl-only_sft-weighted/e1_base/spkl-only-e1_10a78140": 7,
+  "Remek/OpenChat3.5-0106-Spichlerz-Bocian": 7,
+  "alpindale/Mistral-7B-v0.2-hf": 7,
+  "Azurro/APT3-275M-Base": 0.3,
+  "szymonrucinski/Curie-7B-v1": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v4/e0_base/spkl-all-e0-lr5e5_a47a2047": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v4/e0_base/spkl-all-e1_1774eb92": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v4/e0_base/spkl-all-e0-lr2e6_71659188": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v4/e0_base/spkl-all-e0_35239ee5": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v4/e0_base/spkl-all-e2_5257da77": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v4/e0_base/spkl-all-e3_5ca4603b": 7,
+  "speakleash/mistral-apt3-7B/spkl-only_sft_v3/e0_base/spkl-only-e3_90666ab5": 7,
+  "speakleash/mistral-apt3-7B/spkl-only_sft_v3/e0_base/spkl-only-e1_4e524cad": 7,
+  "speakleash/mistral-apt3-7B/spkl-only_sft_v3/e0_base/spkl-only-e0_40cdde38": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v3/e0_base/spkl-all-e0_67274d1b": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v3/e0_base/spkl-all-e1_695e8b44": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v3/e0_base/spkl-all-e2_a9e6a2f0": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v3/e0_base/spkl-all-e3_2ff00c2b": 7,
+  "speakleash/mistral-apt3-7B/spkl_sft_v2/e1_4067e14e": 7,
+  "speakleash/mistral-apt3-7B/spkl_sft_v2/e0_6214300a": 7,
+  "speakleash/mistral-apt3-7B/only-spi_sft_v2/e1_596202b3": 7,
+  "speakleash/mistral-apt3-7B/only-spi_sft_v2/e0_c4ea165e": 7,
+  "speakleash/mistral-apt3-7B/spkl-only_sft_v4/e0_base/spkl-only-e0_c00001c4": 7,
+  "speakleash/mistral-apt3-7B/spkl-only_sft_v4/e0_base/spkl-only-e3_2bcd3961": 7,
+  "speakleash/mistral-apt3-7B/spkl-only_sft_v4/e0_base/spkl-only-e1_f2730438": 7,
+  "speakleash/mistral-apt3-7B/spkl-only_sft_v4/e0_base/spkl-only-e2_f39a22a2": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v3-lr2/e0_base/spkl-all-e0-lr6_376eb1d5": 7,
+  "speakleash/mistral-apt3-7B/spkl-all_sft_v3-lr2/e0_base/spkl-all-e0-lr5_54b6226f": 7,
+  "speakleash/mistral-apt3-7B/spkl-only_sft_v3/e0_base/spkl-only-e2_f036d0fd": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_7e5-e0_e143e6ce": 7,
+  "Nexusflow/Starling-LM-7B-beta": 7,
+  "RWKV/v5-Eagle-7B-HF": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e0_base_2e5/spkl-only-e2_afcfbe2d": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e0_base_2e5/spkl-only-e3_6908149d": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5/spkl-only-e2_d5a874b1": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5/spkl-only-e3_1be744af": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5_v6/spkl-only-e0_4efab00a": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5_v6/spkl-only-e1_1b706f85": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5_v6/spkl-only-e2_f86f7889": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5_v6/spkl-only-e3_13641875": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5_v7w/spkl-only-e0_1f5f4968": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5_v7w/spkl-only-e1_50de9812": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5_v7w/spkl-only-e2_dd38abb9": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5_v7w/spkl-only-e3_36236df3": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5_v8w/spkl-only-e0_e185fb84": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5_v8w/spkl-only-e1_fb5d327f": 7,
+  "speakleash/mistral-apt3-7B_v2/spkl-only_sft/e1_base_2e5_v8w/spkl-only-e2_dd71be08": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft_v2/e1_base/spkl-all_3e6_v8w-e0_d2d8a320": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft_v2/e1_base/spkl-all_3e6_v8w-e1_cd7c61a1": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft_v2/e1_base/spkl-all_v8wa_9e6-e0_32c27aa5": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft_v2/e1_base/spkl-all_v8wa_9e6-e1_518b38ca": 7,
+  "speakleash/mistral_7B-v2/spkl-all_sft_v2/e1_base/spkl-all_v8wa_9e6-e2_84fb05a1": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa_3e6-e0_2ba34bd9": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa_3e6-e1_35ecfaaa": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa_3e6-e2_920b5c3f": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa_7e6-e0_d137146f": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa_7e6-e1_5bddbd74": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa_7e6-e2_bbc67e89": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa_7e6-e2b_53f28c53": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa_7e6-e3_9931f988": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa_7e6-e4_0bc82b61": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v8wa_9e6-e0_8aa4a0ae": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v8wa_9e6-e1_57357d6c": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v8wa_9e6-e2_5eb84913": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v9wa_3e6-e0_ae5e354c": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v9wa_7e6-e0_724b2d41": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v9wa_7e6-e1_d962636b": 7,
+  "speakleash/Bielik-7B-v0.1": 7,
+  "NousResearch/Nous-Hermes-2-SOLAR-10.7B": 10.7,
+  "Qwen/Qwen1.5-7B-Chat": 7,
+  "THUDM/chatglm3-6b-base": 6,
+  "THUDM/chatglm3-6b": 6,
+  "TeeZee/Bielik-SOLAR-LIKE-10.7B-Instruct-v0.1": 10.7,
+  "google/gemma-1.1-2b-it": 2,
+  "meta-llama/Meta-Llama-3-8B-Instruct": 8,
+  "meta-llama/Meta-Llama-3-8B-Instruct,max_length=4096": 8,
+  "meta-llama/Meta-Llama-3-8B": 8,
+  "meta-llama/Meta-Llama-3-8B,max_length=4096": 8,
+  "microsoft/WizardLM-2-7B": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa4_9e6-e0_193ad881": 7,
+  "speakleash/mistral_7B-v2/spkl-only_sft_v2/e1_base/spkl-only_v10wa4_9e6-e1_f40e0808": 7,
+  "speakleash/Bielik-7B-Instruct-v0.1": 7,
+  "speakleash/mistral_7B-v3/spkl-only_sft_v0/e0_base/spkl-only_v11wa_9e6-e0_fe38d62e": 7,
+  "speakleash/mistral_7B-v3/spkl-only_sft_v0/e0_base/spkl-only_v11wa_9e6-e1_6f84698e": 7,
+  "speakleash/mistral_7B-v3/spkl-only_sft_v0/e0_base/spkl-only_v11wap_9e6-e0_5c6927dd": 7,
+  "speakleash/mistral_7B-v3/spkl-only_sft_v0/e0_base/spkl-only_v11wap_9e6-e1_1d6755a9": 7,
+  "speakleash/mistral_7B-v3/spkl-only_v0-e0_b93294c8": 7,
+  "speakleash/mistral_7B-v3/spkl-only_v2-e0_e5547fd5": 7,
+  "speakleash/Bielik-7B-Instruct-v0.1-GPTQ,autogptq=True": 7,
+  "speakleash/Bielik-7B-Instruct-v0.1,load_in_4bit=True": 7,
+  "speakleash/Test-v02-ep3": 7,
+  "speakleash/mistral_7B-v3/spkl-only_v2-e1.34500_a9c75816": 7,
+  "CohereForAI/c4ai-command-r-v01,max_length=4096": 35,
+  "Qwen/Qwen1.5-14B-Chat": 14,
+  "Remek/Llama-3-8B-Omnibus-1-PL-v01-INSTRUCT": 8,
+  "Remek/Llama-3-8B-Omnibus-1-PL-v01-INSTRUCT,max_length=4096": 8,
+  "internlm/internlm2-20b,max_length=4096": 20,
+  "internlm/internlm2-chat-20b,max_length=4096": 20,
+  "lex-hue/Delexa-7b": 7,
+  "lmsys/vicuna-13b-v1.5": 13,
+  "maciek-pioro/Mixtral-8x7B-v0.1-pl,max_length=4096": 46.7,
+  "mistralai/Mixtral-8x7B-Instruct-v0.1,max_length=4096": 46.7,
+  "mistralai/Mixtral-8x7B-v0.1,max_length=4096": 46.7,
+  "speakleash/Test-001-wiki": 7,
+  "speakleash/Test-002": 7,
+  "teknium/OpenHermes-13B": 13,
+  "meta-llama/Meta-Llama-3-70B-Instruct,max_length=4096": 70,
+  "meta-llama/Meta-Llama-3-70B,max_length=4096": 70,
+  "mistralai/Mixtral-8x22B-Instruct-v0.1,max_length=4096": 141,
+  "mistralai/Mixtral-8x22B-v0.1,max_length=4096": 141,
+  "Qwen/Qwen1.5-14B-Chat,max_length=4096": 14,
+  "Qwen/Qwen1.5-32B-Chat,max_length=4096": 32,
+  "Qwen/Qwen1.5-72B-Chat,max_length=4096": 72,
+  "Qwen/Qwen1.5-32B,max_length=4096": 32,
+  "Qwen/Qwen1.5-72B,max_length=4096": 72,
+  "Qwen/Qwen1.5-7B": 7,
+  "Qwen/Qwen2-0.5B-Instruct": 0.5,
+  "Qwen/Qwen2-0.5B": 0.5,
+  "Qwen/Qwen2-1.5B-Instruct": 1.5,
+  "Qwen/Qwen2-1.5B": 1.5,
+  "Qwen/Qwen2-7B-Instruct": 7,
+  "Qwen/Qwen2-7B": 7,
+  "model=gpt-3.5-turbo-instruct": 20,
+  "model=gpt-4-turbo-2024-04-09": 1000,
+  "01-ai/Yi-1.5-6B-Chat": 6,
+  "01-ai/Yi-1.5-6B": 6,
+  "01-ai/Yi-1.5-9B-Chat": 9,
+  "01-ai/Yi-1.5-9B": 9,
+  "CohereForAI/aya-23-35B,max_length=4096": 35,
+  "CohereForAI/aya-23-8B": 8,
+  "NousResearch/Hermes-2-Pro-Llama-3-8B": 8,
+  "NousResearch/Hermes-2-Theta-Llama-3-8B": 8,
+  "Remek/OpenChat-3.5-0106-PL-Omnibusv2": 7,
+  "mistralai/Mistral-7B-Instruct-v0.3": 7,
+  "mistralai/Mistral-7B-v0.3": 7,
+  "nvidia/Llama3-ChatQA-1.5-8B": 8,
+  "openchat/openchat-3.5-0106-gemma": 7,
+  "openchat/openchat-3.6-8b-20240522": 8,
+  "tiiuae/falcon-11B": 11,
+  "mlabonne/NeuralDaredevil-8B-abliterated": 8,
+  "01-ai/Yi-1.5-34B-Chat,max_length=4096": 34,
+  "Qwen/Qwen2-57B-A14B-Instruct,max_length=4096": 57,
+  "Qwen/Qwen2-72B-Instruct,max_length=4096": 72,
+  "Qwen/Qwen2-72B,max_length=4096": 72,
+  "THUDM/glm-4-9b-chat": 9,
+  "THUDM/glm-4-9b": 9,
+  "google/recurrentgemma-9b-it": 9,
+  "microsoft/Phi-3-medium-4k-instruct,max_length=4096": 14,
+  "microsoft/Phi-3-mini-4k-instruct": 3.8,
+  "microsoft/Phi-3-small-8k-instruct": 7.4,
+  "ssmits/Falcon2-5.5B-Polish": 5.5,
+  "alpindale/WizardLM-2-8x22B,max_length=4096": 141,
+  "dreamgen/WizardLM-2-7B": 7
+}