updates

app.py

@@ -6,6 +6,7 @@ import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
+
 from src.assets.css_html_js import custom_css, get_window_url_params
 from src.assets.text_content import (
     CITATION_BUTTON_LABEL,
@@ -232,9 +233,17 @@ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     ]
     return filtered_df
 
+NUMERIC_INTERVALS = {
+    "< 1.5B": (0, 1.5),
+    "~3B": (1.5, 5),
+    "~7B": (6, 11),
+    "~13B": (12, 15),
+    "~35B": (16, 55),
+    "60B+": (55, 10000),
+}
 
 def filter_models(
-    df: pd.DataFrame, current_columns_df: pd.DataFrame, type_query: str, size_query: str, show_deleted: bool
+    df: pd.DataFrame, current_columns_df: pd.DataFrame, type_query: list, size_query: list, show_deleted: bool
 ) -> pd.DataFrame:
     current_columns = current_columns_df.columns
 
@@ -242,26 +251,14 @@ def filter_models(
     if show_deleted:
         filtered_df = df[current_columns]
     else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] is True][current_columns]
-
-    if type_query != "all":
-        type_emoji = type_query[0]
-        filtered_df = filtered_df[df[AutoEvalColumn.model_type_symbol.name] == type_emoji]
-
-    if size_query != "all":
-        numeric_intervals = {
-            "all": (0, 10000),
-            "< 1B": (0, 1),
-            "~3B": (1, 5),
-            "~7B": (6, 11),
-            "~13B": (12, 15),
-            "~35B": (16, 55),
-            "60B+": (55, 10000),
-        }
-        numeric_interval = numeric_intervals[size_query]
-        params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-
-        filtered_df = filtered_df[params_column.between(*numeric_interval)]
+        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True][current_columns]
+
+    type_emoji = [t[0] for t in type_query]
+    filtered_df = filtered_df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+
+    numeric_interval = [NUMERIC_INTERVALS[s] for s in size_query]
+    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+    filtered_df = filtered_df[params_column.between(numeric_interval[0][0], numeric_interval[-1][-1])]
 
     return filtered_df
 
@@ -314,31 +311,27 @@ with demo:
                         elem_id="search-bar",
                     )
                     with gr.Box(elem_id="box-filter"):
-                        filter_columns_type = gr.Radio(
-                            label="⏚ Filter model types",
+                        filter_columns_type = gr.CheckboxGroup(
+                            label="Model types",
                             choices=[
-                                "all",
                                 ModelType.PT.to_str(),
                                 ModelType.FT.to_str(),
                                 ModelType.IFT.to_str(),
                                 ModelType.RL.to_str(),
                             ],
-                            value="all",
+                            value=[
+                                ModelType.PT.to_str(),
+                                ModelType.FT.to_str(),
+                                ModelType.IFT.to_str(),
+                                ModelType.RL.to_str(),
+                            ],
                             interactive=True,
                             elem_id="filter-columns-type",
                         )
-                        filter_columns_size = gr.Radio(
-                            label="⏚ Filter model sizes",
-                            choices=[
-                                "all",
-                                "< 1B",
-                                "~3B",
-                                "~7B",
-                                "~13B",
-                                "~35B",
-                                "60B+",
-                            ],
-                            value="all",
+                        filter_columns_size = gr.CheckboxGroup(
+                            label="Model sizes",
+                            choices=list(NUMERIC_INTERVALS.keys()),
+                            value=list(NUMERIC_INTERVALS.keys()),
                             interactive=True,
                             elem_id="filter-columns-size",
                         )
@@ -497,6 +490,6 @@ with demo:
     )
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(restart_space, "interval", seconds=3600)
 scheduler.start()
 demo.queue(concurrency_count=40).launch()

src/display_models/get_model_metadata.py

@@ -26,6 +26,12 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
             model_data[AutoEvalColumn.likes.name] = None
             model_data[AutoEvalColumn.params.name] = get_model_size(model_name, None)
             continue
+        except Exception as e:
+            print("Repo fetch error", model_name)
+            model_data[AutoEvalColumn.license.name] = None
+            model_data[AutoEvalColumn.likes.name] = None
+            model_data[AutoEvalColumn.params.name] = get_model_size(model_name, None)
+            continue
 
         model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
         model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)

src/display_models/model_metadata_flags.py

@@ -4,6 +4,7 @@ FLAGGED_MODELS = {
     "Voicelab/trurl-2-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/202",
     "deepnight-research/llama-2-70B-inst": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/207",
     "Aspik101/trurl-2-13b-pl-instruct_unload": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/213",
+    "Fredithefish/ReasonixPajama-3B-HF": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/236",
 }
 
 # Models which have been requested by orgs to not be submitted on the leaderboard

src/display_models/model_metadata_type.py

@@ -21,6 +21,8 @@ class ModelType(Enum):
 
 
 MODEL_TYPE_METADATA: Dict[str, ModelType] = {
+    "Qwen/Qwen-7B": ModelType.PT,
+    "Qwen/Qwen-7B-Chat": ModelType.RL,
     "notstoic/PygmalionCoT-7b": ModelType.IFT,
     "aisquared/dlite-v1-355m": ModelType.IFT,
     "aisquared/dlite-v1-1_5b": ModelType.IFT,