Merge branch 'main' of https://huggingface.co/spaces/gsaivinay/open_llm_leaderboard

app.py

@@ -109,6 +109,8 @@ leaderboard_df = original_df.copy()
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(eval_queue, eval_queue_private, EVAL_REQUESTS_PATH, EVAL_COLS)
 
+print(leaderboard_df["Precision"].unique())
+
 
 ## INTERACTION FUNCTIONS
 def add_new_eval(
@@ -211,6 +213,8 @@ def change_tab(query_param: str):
 
 
 # Searching and filtering
+def update_table(hidden_df: pd.DataFrame, current_columns_df: pd.DataFrame, columns: list, type_query: list, precision_query: str, size_query: list, show_deleted: bool, query: str):
+    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
 def update_table(hidden_df: pd.DataFrame, current_columns_df: pd.DataFrame, columns: list, type_query: list, precision_query: str, size_query: list, show_deleted: bool, query: str):
     filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
     if query != "":
@@ -245,6 +249,7 @@ NUMERIC_INTERVALS = {
 
 def filter_models(
     df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
+    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
 ) -> pd.DataFrame:
     # Show all models
     if show_deleted:
@@ -273,6 +278,12 @@ with demo:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
                     with gr.Row():
                         search_bar = gr.Textbox(
                             placeholder=" 🔍 Search for your model and press ENTER...",
@@ -339,6 +350,13 @@ with demo:
                             interactive=True,
                             elem_id="filter-columns-precision",
                         )
+                        filter_columns_precision = gr.CheckboxGroup(
+                            label="Precision",
+                            choices=["torch.float16", "torch.bfloat16", "torch.float32", "8bit", "4bit", "GPTQ"],
+                            value=["torch.float16", "torch.bfloat16", "torch.float32", "8bit", "4bit", "GPTQ"],
+                            interactive=True,
+                            elem_id="filter-columns-precision",
+                        )
                         filter_columns_size = gr.CheckboxGroup(
                             label="Model sizes",
                             choices=list(NUMERIC_INTERVALS.keys()),
@@ -382,6 +400,7 @@ with demo:
                     shown_columns,
                     filter_columns_type,
                     filter_columns_precision,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
                     search_bar,
@@ -396,6 +415,7 @@ with demo:
                     shown_columns,
                     filter_columns_type,
                     filter_columns_precision,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
                     search_bar,
@@ -418,6 +438,22 @@ with demo:
                 leaderboard_table,
                 queue=True,
             )
+            filter_columns_precision.change(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    shown_columns,
+                    filter_columns_type,
+                    filter_columns_precision,
+                    filter_columns_precision,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                    search_bar,
+                ],
+                leaderboard_table,
+                queue=True,
+            )
             filter_columns_precision.change(
                 update_table,
                 [
@@ -441,6 +477,7 @@ with demo:
                     shown_columns,
                     filter_columns_type,
                     filter_columns_precision,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
                     search_bar,
@@ -456,6 +493,7 @@ with demo:
                     shown_columns,
                     filter_columns_type,
                     filter_columns_precision,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
                     search_bar,

src/assets/text_content.py

@@ -1,7 +1,7 @@
 from src.display_models.model_metadata_type import ModelType
 
 TITLE = """<h1 align="center" id="space-title">🤗 Open LLM Leaderboard</h1>
-<h2 align="center" id="space-title">This space displays GPT-4 and GPT-3.5 scores from [techinal paper](https://cdn.openai.com/papers/gpt-4.pdf)</h2>"""
+<h2 align="center" id="space-title">This space displays GPT-4 and GPT-3.5 scores from <a href="https://cdn.openai.com/papers/gpt-4.pdf" target="_blank" rel="noopener noreferrer">techinal paper</a></h2>"""
 
 INTRODUCTION_TEXT = """
 📐 The 🤗 Open LLM Leaderboard aims to track, rank and evaluate open LLMs and chatbots.

src/display_models/get_model_metadata.py

@@ -10,6 +10,8 @@ from huggingface_hub import HfApi
 from tqdm import tqdm
 from transformers import AutoModel, AutoConfig
 from accelerate import init_empty_weights
+from transformers import AutoModel, AutoConfig
+from accelerate import init_empty_weights
 
 from src.display_models.model_metadata_flags import DO_NOT_SUBMIT_MODELS, FLAGGED_MODELS
 from src.display_models.model_metadata_type import MODEL_TYPE_METADATA, ModelType, model_type_from_str
@@ -23,6 +25,7 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
     try:
         with open("model_info_cache.pkl", "rb") as f:
             model_info_cache = pickle.load(f)
+    except (EOFError, FileNotFoundError):
     except (EOFError, FileNotFoundError):
         model_info_cache = {}
     try:
@@ -30,6 +33,11 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
             model_size_cache = pickle.load(f)
     except (EOFError, FileNotFoundError):
         model_size_cache = {}
+    try:
+        with open("model_size_cache.pkl", "rb") as f:
+            model_size_cache = pickle.load(f)
+    except (EOFError, FileNotFoundError):
+        model_size_cache = {}
 
     for model_data in tqdm(leaderboard_data):
         model_name = model_data["model_name_for_query"]
@@ -47,18 +55,26 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
                 if model_name not in model_size_cache:
                     model_size_cache[model_name] = get_model_size(model_name, None)
                 model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
+                if model_name not in model_size_cache:
+                    model_size_cache[model_name] = get_model_size(model_name, None)
+                model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
 
         model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
         model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
         if model_name not in model_size_cache:
             model_size_cache[model_name] = get_model_size(model_name, model_info)
         model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
+        if model_name not in model_size_cache:
+            model_size_cache[model_name] = get_model_size(model_name, model_info)
+        model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
     
     # save cache to disk in pickle format
     with open("model_info_cache.pkl", "wb") as f:
         pickle.dump(model_info_cache, f)
     with open("model_size_cache.pkl", "wb") as f:
         pickle.dump(model_size_cache, f)
+    with open("model_size_cache.pkl", "wb") as f:
+        pickle.dump(model_size_cache, f)
 
 
 def get_model_license(model_info):