refactor: update code for latest Gradio API

Dockerfile

@@ -1,17 +0,0 @@
-FROM python:3.10-slim
-
-ENV DEBIAN_FRONTEND=noninteractive
-WORKDIR /app
-
-RUN apt-get update && apt-get install -y \
-    git git-lfs ffmpeg libsm6 libxext6 libgl1 \
-    && rm -rf /var/lib/apt/lists/* \
-    && git lfs install
-
-RUN pip install --no-cache-dir -U pip setuptools wheel
-
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
-COPY . .
-CMD ["python", "app.py"]

README.md

@@ -3,8 +3,9 @@ title: Low-bit Quantized Open LLM Leaderboard
 emoji: 🏆
 colorFrom: green
 colorTo: indigo
-sdk: docker
-sdk_version: 4.31.5
+sdk: gradio
+sdk_version: 6.5.1
+python_version: 3.11 
 app_file: app.py
 pinned: true
 license: apache-2.0

app.py

@@ -1,5 +1,4 @@
 import os
-
 import gradio as gr
 import pandas as pd
 import re
@@ -24,9 +23,7 @@ from src.display.utils import (
     NUMERIC_INTERVALS,
     NUMERIC_MODELSIZE,
     TYPES,
-    # 改为导入实例
     auto_eval_cols,
-    eval_queue_cols,
     GroupDtype,
     ModelType,
     fields,
@@ -46,21 +43,36 @@ from src.tools.plots import (
     create_plot_df,
     create_scores_df,
 )
-from gradio_modal import Modal
 import plotly.graph_objects as go
 
 selected_indices = []
 selected_values = {}
 selected_dropdown_weight = 'All'
 
+# Start ephemeral Spaces on PRs (see config in README.md)
+#enable_space_ci()
+
 precision_to_dtype = {
-    "2bit": ["int2"], "3bit": ["int3"], "4bit": ["int4", "nf4", "fp4"],
-    "8bit": ["int8"], "16bit": ['float16', 'bfloat16'], "32bit": ["float32"], "?": ["?"],
+    "2bit": ["int2"],
+    "3bit": ["int3"],
+    "4bit": ["int4", "nf4", "fp4"],
+    "8bit": ["int8"],
+    "16bit": ['float16', 'bfloat16'],
+    "32bit": ["float32"],
+    "?": ["?"],
 }
 
 dtype_to_precision = {
-    "int2": ["2bit"], "int3": ["3bit"], "int4": ["4bit"], "nf4": ["4bit"], "fp4": ["4bit"],
-    "int8": ["8bit"], "float16": ["16bit"], "bfloat16": ["16bit"], "float32": ["32bit"], "?": ["?"],
+    "int2": ["2bit"],
+    "int3": ["3bit"],
+    "int4": ["4bit"],
+    "nf4": ["4bit"],
+    "fp4": ["4bit"],
+    "int8": ["8bit"],
+    "float16": ["16bit"],
+    "bfloat16": ["16bit"],
+    "float32": ["32bit"],
+    "?": ["?"],
 }
 
 current_weightDtype = ["int2", "int3", "int4", "nf4", "fp4", "?"]
@@ -68,7 +80,7 @@ current_computeDtype = ['int8', 'bfloat16', 'float16', 'float32']
 current_quant = [t.to_str() for t in QuantType if t != QuantType.QuantType_None]
 current_precision = ['2bit', '3bit', '4bit', '8bit', '?']
 
-# --- 工具函数保持不变 ---
+
 def display_sort(key):
     order = {"All": 0, "?": 1, "int2": 2, "int3": 3, "int4": 4, "fp4": 5, "nf4": 6, "float16": 7, "bfloat16": 8, "float32": 9}
     return order.get(key, float('inf'))
@@ -77,260 +89,758 @@ def comp_display_sort(key):
     order = {"All": 0, "?": 1, "int8": 2, "float16": 3, "bfloat16": 4, "float32": 5}
     return order.get(key, float('inf'))
 
-# --- 更新逻辑保持逻辑不变，仅做属性名适配 ---
 def update_quantization_types(selected_quant):
-    global current_weightDtype, current_computeDtype, current_quant, current_precision
+    global current_weightDtype
+    global current_computeDtype
+    global current_quant
+    global current_precision
+
     if set(current_quant) == set(selected_quant):
-        return [gr.Dropdown(choices=current_weightDtype, value=selected_dropdown_weight),
-                gr.Dropdown(choices=current_computeDtype, value="All"),
-                gr.CheckboxGroup(value=current_precision)]
+        return [
+            gr.Dropdown(choices=current_weightDtype, value=selected_dropdown_weight),
+            gr.Dropdown(choices=current_computeDtype, value="All"),
+            gr.CheckboxGroup(value=current_precision),
+        ]
+     
+    # print('update_quantization_types', selected_quant, current_quant)
     if any(value != '✖ None' for value in selected_quant):
         selected_weight = ['All', '?', 'int2', 'int3', 'int4', 'nf4', 'fp4', 'int8']
         selected_compute = ['All', '?', 'int8', 'float16', 'bfloat16', 'float32']
         selected_precision = ["2bit", "3bit", "4bit", "8bit", "?"]
-    current_weightDtype, current_computeDtype, current_quant, current_precision = selected_weight, selected_compute, selected_quant, selected_precision
-    return [gr.Dropdown(choices=selected_weight, value="All"),
-            gr.Dropdown(choices=selected_compute, value="All"),
-            gr.CheckboxGroup(value=selected_precision)]
+    
+    current_weightDtype = selected_weight
+    current_computeDtype = selected_compute
+    current_quant = selected_quant  
+    current_precision = selected_precision
+
+    return [
+        gr.Dropdown(choices=selected_weight, value="All"),
+        gr.Dropdown(choices=selected_compute, value="All"),
+        gr.CheckboxGroup(value=selected_precision),
+    ]
 
 def update_Weight_Precision(temp_precisions):
-    global current_weightDtype, current_computeDtype, current_quant, current_precision, selected_dropdown_weight
+    global current_weightDtype
+    global current_computeDtype
+    global current_quant
+    global current_precision
+    global selected_dropdown_weight
+
+    # print('temp_precisions', temp_precisions)
     if set(current_precision) == set(temp_precisions):
-        return [gr.Dropdown(choices=current_weightDtype, value=selected_dropdown_weight),
-                gr.Dropdown(choices=current_computeDtype, value="All"),
-                gr.CheckboxGroup(value=current_precision),
-                gr.CheckboxGroup(value=current_quant)]
-    selected_weight, selected_compute = [], ['All', '?', 'int8', 'float16', 'bfloat16', 'float32'] 
+        return [
+            gr.Dropdown(choices=current_weightDtype, value=selected_dropdown_weight),
+            gr.Dropdown(choices=current_computeDtype, value="All"),
+            gr.CheckboxGroup(value=current_precision),
+            gr.CheckboxGroup(value=current_quant),
+        ]   # No update needed
+    
+    selected_weight = []
+    selected_compute = ['All', '?', 'int8', 'float16', 'bfloat16', 'float32'] 
     selected_quant = [t.to_str() for t in QuantType if t != QuantType.QuantType_None]
+
     if temp_precisions[-1] in ["16bit", "32bit"]:
         selected_precisions = [p for p in temp_precisions if p in ["16bit", "32bit"]]
     else:
         selected_precisions = [p for p in temp_precisions if p not in ["16bit", "32bit"]]
+
     current_precision = list(set(selected_precisions))
-    if len(current_precision) > 1 or (selected_dropdown_weight != 'All' and set(dtype_to_precision.get(selected_dropdown_weight, [])) != set(current_precision)):
+    # print('selected_dropdown_weight', selected_dropdown_weight)
+
+    if len(current_precision) > 1:
         selected_dropdown_weight = 'All'
+    elif selected_dropdown_weight != 'All' and set(dtype_to_precision[selected_dropdown_weight]) != set(current_precision):
+        selected_dropdown_weight = 'All'
+
+    # print('final', current_precision)
+    # Map selected_precisions to corresponding weights
     for precision in current_precision:
-        if precision in precision_to_dtype: selected_weight.extend(precision_to_dtype[precision])
+        if precision in precision_to_dtype:
+            selected_weight.extend(precision_to_dtype[precision])
+    
+    # Special rules for 16bit and 32bit
     if "16bit" in current_precision:
-        selected_weight = [o for o in selected_weight if o in ["All", "?", "float16", "bfloat16"]]
-        if "int8" in selected_compute: selected_compute.remove("int8")
+        selected_weight = [option for option in selected_weight if option in ["All", "?", "float16", "bfloat16"]]
+        if "int8" in selected_compute:
+            selected_compute.remove("int8")
+                    
     if "32bit" in current_precision:
-        selected_weight = [o for o in selected_weight if o in ["All", "?", "float32"]]
-        if "int8" in selected_compute: selected_compute.remove("int8")
-    if "16bit" in current_precision or "32bit" in current_precision: selected_quant = ['✖ None']
-    selected_weight = list(set(["All", "?"] + selected_weight))
-    selected_compute = list(set(["All", "?"] + selected_compute))
-    current_weightDtype, current_computeDtype, current_quant = selected_weight, selected_compute, selected_quant
-    return [gr.Dropdown(choices=selected_weight, value=selected_dropdown_weight),
-            gr.Dropdown(choices=selected_compute, value="All"),
-            gr.CheckboxGroup(value=selected_precisions),
-            gr.CheckboxGroup(value=selected_quant)]
+        selected_weight = [option for option in selected_weight if option in ["All", "?", "float32"]]
+        if "int8" in selected_compute:
+            selected_compute.remove("int8")
+
+    if "16bit" in current_precision or "32bit" in current_precision:
+        selected_quant = ['✖ None']
+    if "16bit" in current_precision and "32bit" in current_precision:
+        selected_weight = ["All", "?", "float16", "bfloat16", "float32"]        
+    # Ensure "All" and "?" options are included
+    selected_weight = ["All", "?"] + [opt for opt in selected_weight if opt not in ["All", "?"]]
+    selected_compute = ["All", "?"] + [opt for opt in selected_compute if opt not in ["All", "?"]]
+    
+    # Remove duplicates
+    selected_weight = list(set(selected_weight))
+    selected_compute = list(set(selected_compute))
+    
+    # Update global variables
+    current_weightDtype = selected_weight
+    current_computeDtype = selected_compute
+    current_quant = selected_quant          
+    
+    # Return updated components
+    return [
+        gr.Dropdown(choices=selected_weight, value=selected_dropdown_weight),
+        gr.Dropdown(choices=selected_compute, value="All"),
+        gr.CheckboxGroup(value=selected_precisions),
+        gr.CheckboxGroup(value=selected_quant),
+    ]
 
 def update_Weight_Dtype(weight):    
     global selected_dropdown_weight
-    if weight == selected_dropdown_weight or weight == 'All': return current_precision
-    selected_precisions = dtype_to_precision.get(weight, [])
-    selected_dropdown_weight = weight       
+    # print('update_Weight_Dtype', weight)
+    # Initialize selected_precisions
+    if weight == selected_dropdown_weight or weight == 'All':
+        return current_precision
+    else:
+        selected_precisions = []        
+        selected_precisions.extend(dtype_to_precision[weight])
+    selected_dropdown_weight =  weight       
+    # print('selected_precisions', selected_precisions)
+    # Return updated components
     return selected_precisions
 
+
+
+
 def restart_space():
     API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
 
+
 def init_space(full_init: bool = True):
+    
     if full_init:
         try:
             branch = REPO.active_branch.name
             REPO.remotes.origin.pull(branch)
-            snapshot_download(repo_id=DYNAMIC_INFO_REPO, local_dir=DYNAMIC_INFO_PATH, repo_type="dataset", etag_timeout=30)
-        except Exception as e: print(str(e)); restart_space()
-    
-    raw_data, original_df = get_leaderboard_df(GIT_RESULTS_PATH, GIT_STATUS_PATH, DYNAMIC_INFO_FILE_PATH, COLS, BENCHMARK_COLS)
-    
-    # 防御补全：如果没数据也要有骨架，防止 KeyError 'Model'
-    if original_df.empty:
-        original_df = pd.DataFrame(columns=[c.name for c in fields(auto_eval_cols)])
+        except Exception as e:
+            # print(str(e))
+            restart_space()
 
+        try:
+            # print(DYNAMIC_INFO_PATH)
+            snapshot_download(
+                repo_id=DYNAMIC_INFO_REPO, local_dir=DYNAMIC_INFO_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+            )
+        except Exception:
+            restart_space()
+
+    raw_data, original_df = get_leaderboard_df(
+        results_path=GIT_RESULTS_PATH, 
+        requests_path=GIT_STATUS_PATH, 
+        dynamic_path=DYNAMIC_INFO_FILE_PATH, 
+        cols=COLS, 
+        benchmark_cols=BENCHMARK_COLS
+    )
+    # update_collections(original_df.copy())
     leaderboard_df = original_df.copy()
+
     plot_df = create_plot_df(create_scores_df(raw_data))
-    (f_q, r_q, p_q) = get_evaluation_queue_df(GIT_STATUS_PATH, EVAL_COLS)
-    return leaderboard_df, original_df, plot_df, f_q, r_q, p_q
+
+    (
+        finished_eval_queue_df,
+        running_eval_queue_df,
+        pending_eval_queue_df,
+    ) = get_evaluation_queue_df(GIT_STATUS_PATH, EVAL_COLS)
+
+    return leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
 
 leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
 
 def str_to_bool(value):
-    return str(value).lower() == "true"
-
-def update_table(hidden_df, columns, type_query, precision_query, size_query, params_query, hide_models, query, compute_dtype, weight_dtype, double_quant, group_dtype):
-    global current_weightDtype, current_computeDtype
-    w_dt = current_weightDtype if weight_dtype in [['All'], 'All'] else [weight_dtype]
-    c_dt = current_computeDtype if compute_dtype == 'All' else [compute_dtype]
-    try: g_dt = [int(group_dtype)] if group_dtype != 'All' else [-1, 1024, 256, 128, 64, 32]
-    except: g_dt = [-1]
-    dq = [True, False] if double_quant == 'All' else [str_to_bool(double_quant)] 
+    if str(value).lower() == "true":
+        return True
+    elif str(value).lower() == "false":
+        return False
+    else:
+        return False
+
+# Searching and filtering
+def update_table(
+    hidden_df: pd.DataFrame,
+    columns: list,
+    type_query: list,
+    precision_query: str,
+    size_query: list,
+    params_query: list,
+    hide_models: list,
+    query: str,
+    compute_dtype: str,
+    weight_dtype: str,
+    double_quant: str,
+    group_dtype: str
+):
+    global init_select
+    global current_weightDtype
+    global current_computeDtype
+
+    if weight_dtype == ['All'] or weight_dtype == 'All':
+        weight_dtype = current_weightDtype
+    else:
+        weight_dtype = [weight_dtype]
+
+    if compute_dtype == 'All':
+        compute_dtype = current_computeDtype
+    else:
+        compute_dtype = [compute_dtype]   
+        
+    if group_dtype == 'All':
+        group_dtype = [-1, 1024, 256, 128, 64, 32]
+    else:
+        try:
+            group_dtype = [int(group_dtype)]
+        except ValueError:
+            group_dtype = [-1]
+
+    if double_quant == 'All':
+        double_quant = [True, False]
+    else:
+        double_quant = [str_to_bool(double_quant)] 
         
-    filtered_df = filter_models(hidden_df, type_query, size_query, params_query, precision_query, hide_models, c_dt, w_dt, dq, g_dt)
+    filtered_df = filter_models(df=hidden_df, type_query=type_query, size_query=size_query, precision_query=precision_query, hide_models=hide_models, compute_dtype=compute_dtype, weight_dtype=weight_dtype, double_quant=double_quant, group_dtype=group_dtype, params_query=params_query)
     filtered_df = filter_queries(query, filtered_df)
-    return select_columns(filtered_df, columns)
+    df = select_columns(filtered_df, columns)
+    return df
+
+
+def load_query(request: gr.Request):  # triggered only once at startup => read query parameter if it exists
+    query = request.query_params.get("query") or ""
+    return query, query # return one for the "search_bar", one for a hidden component that triggers a reload only if value has changed
+
 
 def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
-    if auto_eval_cols.dummy.name not in df.columns: return df
     return df[(df[auto_eval_cols.dummy.name].str.contains(query, case=False))]
 
+
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     always_here_cols = [c.name for c in fields(auto_eval_cols) if c.never_hidden]
     dummy_col = [auto_eval_cols.dummy.name]
-    # 动态取列，解决 KeyError
-    req_cols = always_here_cols + [c for c in COLS if c in df.columns and c in columns] + dummy_col
-    return df[[c for c in req_cols if c in df.columns]]
+    # We use COLS to maintain sorting
+    filtered_df = df[
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + dummy_col
+    ]
+    return filtered_df
+
 
 def filter_queries(query: str, filtered_df: pd.DataFrame):
-    if query == "": return filtered_df
+    """Added by Abishek"""
     final_df = []
-    for _q in [q.strip() for q in query.split(";") if q.strip() != ""]:
-        temp = search_table(filtered_df, _q)
-        if len(temp) > 0: final_df.append(temp)
-    if not final_df: return filtered_df
-    return pd.concat(final_df).drop_duplicates(subset=[auto_eval_cols.model.name, auto_eval_cols.precision.name, auto_eval_cols.revision.name])
-
-def filter_models(df, type_query, size_query, params_query, precision_query, hide_models, compute_dtype, weight_dtype, double_quant, group_dtype):
-    f_df = df.copy()
-    # 增加列存在性检查，防止 KeyError
-    check_cols = {
-        auto_eval_cols.still_on_hub.name: lambda d: d[d[auto_eval_cols.still_on_hub.name] == True] if "Private or deleted" in hide_models else d,
-        auto_eval_cols.merged.name: lambda d: d[d[auto_eval_cols.merged.name] == False] if "Contains a merge/moerge" in hide_models else d,
-        auto_eval_cols.moe.name: lambda d: d[d[auto_eval_cols.moe.name] == False] if "MoE" in hide_models else d,
-        auto_eval_cols.flagged.name: lambda d: d[d[auto_eval_cols.flagged.name] == False] if "Flagged" in hide_models else d,
-    }
-    for col, func in check_cols.items():
-        if col in f_df.columns: f_df = func(f_df)
-
-    type_emoji = [t[0] for t in type_query if t]
-    type_emoji = [e for e in type_emoji if e != '✖'] if any(e != '✖' for e in type_emoji) else ['✖']
-
-    if auto_eval_cols.model_type_symbol.name in f_df.columns:
-        f_df = f_df[f_df[auto_eval_cols.model_type_symbol.name].isin(type_emoji)]
-    if auto_eval_cols.precision.name in f_df.columns:
-        f_df = f_df[f_df[auto_eval_cols.precision.name].isin(precision_query + ["None"])]
-    if auto_eval_cols.weight_dtype.name in f_df.columns:
-        f_df = f_df[f_df[auto_eval_cols.weight_dtype.name].isin(weight_dtype)]
-    
-    # 参数量数值区间过滤
-    if auto_eval_cols.params.name in f_df.columns:
-        numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
-        params_col = pd.to_numeric(f_df[auto_eval_cols.params.name], errors="coerce")
-        f_df = f_df[params_col.apply(lambda x: any(numeric_interval.contains(x)) if pd.notnull(x) else False)]
+    if query != "":
+        queries = [q.strip() for q in query.split(";")]
+        for _q in queries:
+            _q = _q.strip()
+            if _q != "":
+                temp_filtered_df = search_table(filtered_df, _q)
+                if len(temp_filtered_df) > 0:
+                    final_df.append(temp_filtered_df)
+        if len(final_df) > 0:
+            filtered_df = pd.concat(final_df)
+            filtered_df = filtered_df.drop_duplicates(
+                subset=[auto_eval_cols.model.name, auto_eval_cols.precision.name, auto_eval_cols.revision.name]
+            )
+
+    return filtered_df
+
+
+def filter_models(
+    df: pd.DataFrame, type_query: list, size_query: list, params_query:list, precision_query: list, hide_models: list, compute_dtype: list, weight_dtype: list, double_quant: list, group_dtype: list,
+ ) -> pd.DataFrame:
+    # Show all models
+    if "Private or deleted" in hide_models:
+        filtered_df = df[df[auto_eval_cols.still_on_hub.name] == True]
+    else:
+        filtered_df = df
+
+    if "Contains a merge/moerge" in hide_models:
+        filtered_df = filtered_df[filtered_df[auto_eval_cols.merged.name] == False]
+
+    if "MoE" in hide_models:
+        filtered_df = filtered_df[filtered_df[auto_eval_cols.moe.name] == False]
+
+    if "Flagged" in hide_models:
+        filtered_df = filtered_df[filtered_df[auto_eval_cols.flagged.name] == False]
+
+    type_emoji = [t[0] for t in type_query]
+    if any(emoji != '✖' for emoji in type_emoji):
+        type_emoji = [emoji for emoji in type_emoji if emoji != '✖']
+    else:
+        type_emoji = ['✖']
+
+    filtered_df = filtered_df.loc[df[auto_eval_cols.model_type_symbol.name].isin(type_emoji)]
+    filtered_df = filtered_df.loc[df[auto_eval_cols.precision.name].isin(precision_query + ["None"])]
+
+    filtered_df = filtered_df.loc[df[auto_eval_cols.weight_dtype.name].isin(weight_dtype)]
+
+    filtered_df = filtered_df.loc[df[auto_eval_cols.compute_dtype.name].isin(compute_dtype)]
+
+    filtered_df = filtered_df.loc[df[auto_eval_cols.double_quant.name].isin(double_quant)]
+
+    filtered_df = filtered_df.loc[df[auto_eval_cols.group_size.name].isin(group_dtype)]
+
+    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
+    params_column = pd.to_numeric(df[auto_eval_cols.params.name], errors="coerce")
+    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+    filtered_df = filtered_df.loc[mask]
 
-    return f_df
+    numeric_interval_params = pd.IntervalIndex(sorted([NUMERIC_MODELSIZE[s] for s in params_query]))
+    params_column_params = pd.to_numeric(df[auto_eval_cols.model_size.name], errors="coerce")
+    mask_params = params_column_params.apply(lambda x: any(numeric_interval_params.contains(x)))
+    filtered_df = filtered_df.loc[mask_params]
+
+    return filtered_df
 
 def select(df, data: gr.SelectData):
-    global selected_indices, selected_values
+    global selected_indices
+    global selected_values
+    
     selected_index = data.index[0]
-    value = df.iloc[selected_index].iloc[1]
-    match = re.search(r'<a[^>]+>([^<]+)</a>', value)
-    if not match: return gr.CheckboxGroup(list(selected_values.keys()), value=list(selected_values.keys()))
-    text_content = match.group(1)
     if selected_index in selected_indices:
         selected_indices.remove(selected_index)
-        if text_content in selected_values: del selected_values[text_content]
+        
+        value = df.iloc[selected_index].iloc[1]
+        pattern = r'<a[^>]+>([^<]+)</a>'
+        match = re.search(pattern, value)
+        if match:
+            text_content = match.group(1)
+            if text_content in selected_values:
+                del selected_values[text_content]
     else:
         selected_indices.append(selected_index)
-        selected_values[text_content] = value
+
+        value = df.iloc[selected_index].iloc[1]
+        pattern = r'<a[^>]+>([^<]+)</a>'
+        match = re.search(pattern, value)
+        if match:
+            text_content = match.group(1)
+            selected_values[text_content] = value
+
     return gr.CheckboxGroup(list(selected_values.keys()), value=list(selected_values.keys()))
 
+def init_comparison_data():
+    global selected_values
+    return gr.CheckboxGroup(list(selected_values.keys()), value=list(selected_values.keys())) 
+
+def remove_html_tags(value):
+    if isinstance(value, str):
+        return re.sub(r'<[^>]*>', '', value) 
+    return value    
+
+def show_modal():
+    return gr.update(visible=True, elem_classes="custom-modal")
+
+def close_modal_logic():
+    return gr.update(visible=False, elem_classes="modal-hidden")
+
 def generate_spider_chart(df, selected_keys):
     global selected_values
-    current_sel = [selected_values[key] for key in selected_keys if key in selected_values]
-    selected_rows = df[df.iloc[:, 1].isin(current_sel)]
-    cleaned_rows = selected_rows.applymap(lambda x: re.sub(r'<[^>]*>', '', x) if isinstance(x, str) else x)
+    current_selected_values = [selected_values[key] for key in selected_keys if key in selected_values]
+    selected_rows = df[df.iloc[:, 1].isin(current_selected_values)]
+    cleaned_rows = selected_rows.map(remove_html_tags)
+
+
     fig = go.Figure()
-    # 强制指定指标列
-    metrics = ['Average ⬆️', 'ARC-c', 'ARC-e', 'Boolq', 'HellaSwag', 'Lambada', 'MMLU', 'Openbookqa', 'Piqa', 'Truthfulqa', 'Winogrande']
     for _, row in selected_rows.iterrows():
         fig.add_trace(go.Scatterpolar(
-            r=[row.get(m, 0) for m in metrics],
-            theta=metrics, fill='toself', name=re.sub(r'<[^>]*>', '', str(row.get('Model', 'Unknown')))
+            r=[row['Average ⬆️'], row['ARC-c'], row['ARC-e'], row['Boolq'], row['HellaSwag'], row['Lambada'], row['MMLU'], row['Openbookqa'], row['Piqa'], row['Truthfulqa'], row['Winogrande']],
+            theta=['Average ⬆️', 'ARC-c', 'ARC-e', 'Boolq', 'HellaSwag', 'Lambada', 'MMLU', 'Openbookqa', 'Piqa', 'Truthfulqa', 'Winogrande'],
+            fill='toself',
+            name=str(row['Model'])  
         ))
-    fig.update_layout(polar=dict(radialaxis=dict(visible=False)), showlegend=True)
-    return fig, cleaned_rows
+    fig.update_layout(
+        polar=dict(
+            radialaxis=dict(
+                visible=False,
+            )),
+        showlegend=True,
+        margin=dict(l=50, r=50, t=50, b=50), 
+        height=400,
+        autosize=True
+    )
+    
+    return fig, cleaned_rows    
+
+leaderboard_df = filter_models(
+    df=leaderboard_df, 
+    type_query=[t.to_str(" : ") for t in QuantType if t != QuantType.QuantType_None], 
+    size_query=list(NUMERIC_INTERVALS.keys()), 
+    params_query=list(NUMERIC_MODELSIZE.keys()),
+    precision_query=[i.value.name for i in Precision],
+    hide_models=["Private or deleted", "Contains a merge/moerge", "Flagged"], # Deleted, merges, flagged, MoEs,
+    compute_dtype=[i.value.name for i in ComputeDtype],
+    weight_dtype=[i.value.name for i in WeightDtype],
+    double_quant=[True, False],
+    group_dtype=[-1, 1024, 256, 128, 64, 32]
+)
+
 
-# --- 构建界面 ---
-demo = gr.Blocks(css=custom_css)
+demo = gr.Blocks(fill_width=True)
 with demo:
+
+    with gr.Column(elem_classes="custom-modal", visible=False, elem_id="my-modal-container") as modal_window:
+        with gr.Column(elem_classes="modal-content"):            
+            with gr.Column():
+                comparison_plot_inside = gr.Plot()
+            comparison_df_inside = gr.Dataframe(interactive=False)
+            
+            close_btn = gr.Button("Close", variant="primary")
+
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", id=0):
+        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 with gr.Column():
-                    search_bar = gr.Textbox(placeholder=" 🔍 Search model...", show_label=False)
-                    shown_columns = gr.CheckboxGroup(
-                        choices=[c.name for c in fields(auto_eval_cols) if not c.hidden and not c.never_hidden and not c.dummy],
-                        value=[c.name for c in fields(auto_eval_cols) if c.displayed_by_default and not c.hidden and not c.never_hidden],
-                        label="Select columns"
+                    with gr.Row(variant="compact"):
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[
+                                c.name
+                                for c in fields(auto_eval_cols)
+                                if not c.hidden and not c.never_hidden and not c.dummy
+                            ],
+                            value=[
+                                c.name
+                                for c in fields(auto_eval_cols)
+                                if c.displayed_by_default and not c.hidden and not c.never_hidden
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+
+                    with gr.Row():
+                        filter_columns_parameters = gr.CheckboxGroup(
+                        label="Model parameters (in billions of parameters)",
+                        choices=list(NUMERIC_INTERVALS.keys()),
+                        value=list(NUMERIC_INTERVALS.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
+                    )
+                    with gr.Row():
+                        filter_columns_size = gr.CheckboxGroup(
+                        label="Model sizes (GB, int4)",
+                        choices=list(NUMERIC_MODELSIZE.keys()),
+                        value=list(NUMERIC_MODELSIZE.keys()),
+                        interactive=True,
+                        elem_id="filter-columns-size",
                     )
                 with gr.Column(min_width=320):
-                    filter_columns_type = gr.CheckboxGroup(label="Quantization types", choices=[t.to_str() for t in QuantType if t != QuantType.QuantType_None], value=[t.to_str() for t in QuantType if t != QuantType.QuantType_None])
-                    filter_columns_precision = gr.CheckboxGroup(label="Weight precision", choices=[i.value.name for i in Precision], value=[i.value.name for i in Precision if i.value.name not in ['16bit', '32bit']])
-                    with gr.Group():
-                        gr.HTML("<p style='padding: 0.7rem; background: #fff; margin: 0; color: #6b7280;'>Quantization config</p>")
+                    #with gr.Box(elem_id="box-filter"):
+                    filter_columns_type = gr.CheckboxGroup(
+                        label="Quantization types",
+                        choices=[t.to_str() for t in QuantType if t != QuantType.QuantType_None],
+                        value=[t.to_str() for t in QuantType if t != QuantType.QuantType_None],
+                        interactive=True,
+                        elem_id="filter-columns-type",  
+                    )
+                    filter_columns_precision = gr.CheckboxGroup(
+                        label="Weight precision",
+                        choices=[i.value.name for i in Precision],
+                        value=[i.value.name for i in Precision  if ( i.value.name != '16bit' and i.value.name != '32bit')],
+                        interactive=True,
+                        elem_id="filter-columns-precision",
+                    )
+                    with gr.Column(elem_id="quant-config-container") as config: 
+                        gr.HTML("<div class='quant-config-header'>Quantization config</div>")
                         with gr.Row():
-                            f_compute = gr.Dropdown(choices=[i.value.name for i in ComputeDtype], label="Compute Dtype", value="All")
-                            f_weight = gr.Dropdown(choices=[i.value.name for i in WeightDtype], label="Weight Dtype", value="All")
-                            f_double = gr.Dropdown(choices=["All", "True", "False"], label="Double Quant", value="All")
-                            f_group = gr.Dropdown(choices=[i.value.name for i in GroupDtype], label="Group Size", value="All")
+                            filter_columns_computeDtype = gr.Dropdown(choices=[i.value.name for i in ComputeDtype], label="Compute Dtype", multiselect=False, value="All", interactive=True,)
+                            filter_columns_weightDtype = gr.Dropdown(choices=[i.value.name for i in WeightDtype], label="Weight Dtype", multiselect=False, value="All", interactive=True,)
+                            filter_columns_doubleQuant = gr.Dropdown(choices=["All", "True", "False"], label="Double Quant", multiselect=False, value="All", interactive=True)
+                            filter_columns_groupDtype = gr.Dropdown(choices=[i.value.name for i in GroupDtype], label="Group Size", multiselect=False, value="All", interactive=True,)
+
+                    with gr.Row():
+                        with gr.Column(scale=4):
+                            model_comparison = gr.CheckboxGroup(label="Accuracy Comparison (Selected Models from Table)", choices=list(selected_values.keys()), value=list(selected_values.keys()), interactive=True, elem_id="model_comparison")
+                        with gr.Column(scale=1, min_width=150):
+                            spider_btn = gr.Button("Compare", variant="primary", elem_id="compare-button-full")                 
+      
+            never_hidden_cols = [c.name for c in fields(auto_eval_cols) if c.never_hidden]
+
+            user_cols = shown_columns.value
+
+            if len(user_cols) > 0:
+                first_user_col = [user_cols[0]]     
+                remaining_user_cols = user_cols[1:]
+                
+                final_cols = first_user_col + never_hidden_cols + remaining_user_cols
+            else:
+                final_cols = never_hidden_cols
+
+            leaderboard_table = gr.components.Dataframe(
+                value=leaderboard_df[final_cols + [auto_eval_cols.dummy.name]],
+                headers=final_cols,
+                datatype="markdown",
+                elem_id="leaderboard-table",
+                interactive=False,
+                visible=True,
+            )      
+
+            # with gr.BrowserModal(visible=False) as modal:
+            #     map = gr.Plot()
+            #     data_table = gr.Dataframe()
+            #     gr.Column([map, data_table])
             
-            model_comparison = gr.CheckboxGroup(label="Accuracy Comparison", choices=list(selected_values.keys()))
-            spider_btn = gr.Button("Compare")
-
-            # 对齐初始化列
-            init_req = [c.name for c in fields(auto_eval_cols) if c.never_hidden] + shown_columns.value + [auto_eval_cols.dummy.name]
-            init_act = [c for c in init_req if c in leaderboard_df.columns]
-            if not init_act: init_act = [auto_eval_cols.model.name]
-
-            leaderboard_table = gr.Dataframe(
-                value=leaderboard_df[init_act],
-                headers=init_act,
-                datatype=TYPES, interactive=False,
-                column_count=(len(init_act), "fixed")
+            leaderboard_table.select(select, leaderboard_table, model_comparison)
+            spider_btn.click(
+                fn=show_modal,
+                outputs=modal_window
+            ).then(
+                fn=generate_spider_chart,
+                inputs=[leaderboard_table, model_comparison],
+                outputs=[comparison_plot_inside, comparison_df_inside]
+            )
+            close_btn.click(
+                fn=close_modal_logic,
+                outputs=modal_window
+            )
+            demo.load(init_comparison_data, None, model_comparison)
+            
+            if "Weight type" not in original_df.columns:
+                original_df["Weight type"] = "Unknown"
+
+            # Dummy leaderboard for handling the case when the user uses backspace key
+            hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                value=original_df[COLS],
+                headers=COLS,
+                datatype=TYPES,
+                visible=False,
             )
 
-            with Modal(visible=False) as modal:
-                map_p = gr.Plot()
-                data_table = gr.Dataframe()
+            hide_models = gr.Textbox(
+                            placeholder="",
+                            show_label=False,
+                            elem_id="search-bar",
+                            value="",
+                            visible=False,
+
+                        )
             
-            leaderboard_table.select(select, leaderboard_table, model_comparison)
-            spider_btn.click(generate_spider_chart, [leaderboard_table, model_comparison], [map_p, data_table])
-            spider_btn.click(lambda: Modal(visible=True), None, modal)
+            search_bar.submit(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    filter_columns_type,
+                    filter_columns_precision,
+                    filter_columns_parameters,
+                    filter_columns_size,
+                    hide_models,
+                    search_bar,
+                    filter_columns_computeDtype,
+                    filter_columns_weightDtype,
+                    filter_columns_doubleQuant,
+                    filter_columns_groupDtype
+                ],
+                leaderboard_table,
+            )
 
-            hidden_leaderboard = gr.Dataframe(value=original_df[COLS] if set(COLS).issubset(original_df.columns) else original_df, visible=False)
-            search_bar.submit(update_table, [hidden_leaderboard, shown_columns, filter_columns_type, filter_columns_precision, gr.State([]), gr.State([]), gr.State([]), search_bar, f_compute, f_weight, f_double, f_group], leaderboard_table)
+            """
+           
+            # Define a hidden component that will trigger a reload only if a query parameter has been set
+            hidden_search_bar = gr.Textbox(value="", visible=False)
+            hidden_search_bar.change(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    shown_columns,
+                    filter_columns_type,
+                    filter_columns_precision,
+                    filter_columns_size,
+                    hide_models,
+                    search_bar,
+                ],
+                leaderboard_table,
+            )
+            # Check query parameter once at startup and update search bar + hidden component
+            demo.load(load_query, inputs=[], outputs=[search_bar, hidden_search_bar])
+            
+            """
+            filter_columns_type.change(
+                update_quantization_types,
+                [filter_columns_type],
+                [filter_columns_weightDtype, filter_columns_computeDtype, filter_columns_precision]
+            )
+
+            filter_columns_precision.change(
+                update_Weight_Precision,
+                [filter_columns_precision],
+                [filter_columns_weightDtype, filter_columns_computeDtype, filter_columns_precision, filter_columns_type]
+            )
+
+            filter_columns_weightDtype.change(
+                update_Weight_Dtype,
+                [filter_columns_weightDtype],
+                [filter_columns_precision]
+            )
+            # filter_columns_computeDtype.change(
+            #     Compute_Dtype_update,
+            #     [filter_columns_computeDtype, filter_columns_precision],
+            #     [filter_columns_precision, filter_columns_type]
+            # )
+            
+
+    
+            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, filter_columns_parameters, hide_models, filter_columns_computeDtype, filter_columns_weightDtype, filter_columns_doubleQuant, filter_columns_groupDtype]:
+                selector.change(
+                    update_table,
+                    [
+                        hidden_leaderboard_table_for_search,
+                        shown_columns,
+                        filter_columns_type,
+                        filter_columns_precision,
+                        filter_columns_parameters,
+                        filter_columns_size,
+                        hide_models,
+                        search_bar,
+                        filter_columns_computeDtype,
+                        filter_columns_weightDtype,
+                        filter_columns_doubleQuant,
+                        filter_columns_groupDtype
+                    ],
+                    leaderboard_table,
+                    queue=True,
+                )
+
+
+        with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=2):
+            with gr.Row():
+                with gr.Column():
+                    chart = create_metric_plot_obj(
+                        plot_df,
+                        [auto_eval_cols.average.name],
+                        title="Average of Top Scores and Human Baseline Over Time (from last update)",
+                    )
+                    gr.Plot(value=chart, min_width=500) 
+                with gr.Column():
+                    chart = create_metric_plot_obj(
+                        plot_df,
+                        BENCHMARK_COLS,
+                        title="Top Scores and Human Baseline Over Time (from last update)",
+                    )
+                    gr.Plot(value=chart, min_width=500) 
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("❗FAQ", elem_id="llm-benchmark-tab-table", id=4):
+            gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("🚀 Submit ", elem_id="llm-benchmark-tab-table", id=5):
+            with gr.Column():
+                with gr.Row():
+                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 
-        with gr.TabItem("📈 Metrics through time", id=2):
             with gr.Row():
-                gr.Plot(value=create_metric_plot_obj(plot_df, [auto_eval_cols.average.name], title="Average Over Time"))
-                gr.Plot(value=create_metric_plot_obj(plot_df, BENCHMARK_COLS, title="Benchmarks Over Time"))
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name")
+                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+                    private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
+
+                with gr.Column():
+                    """
+                    precision = gr.Dropdown(
+                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
+                        label="Precision",
+                        multiselect=False,
+                        value="4bit",
+                        interactive=True,
+                    )
+                    weight_type = gr.Dropdown(
+                        choices=[i.value.name for i in WeightDtype],
+                        label="Weights dtype",
+                        multiselect=False,
+                        value="int4",
+                        interactive=True,
+                    )
+                    """
+                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)",
+                            visible=not IS_PUBLIC)
+                    compute_type = gr.Dropdown(
+                        choices=[i.value.name for i in ComputeDtype if i.value.name != "All"],
+                        label="Compute dtype",
+                        multiselect=False,
+                        value="float16",
+                        interactive=True,
+                    )
+
+            submit_button = gr.Button("Submit Eval")
+            submission_result = gr.Markdown()
+            submit_button.click(
+                add_new_eval,
+                [
+                    model_name_textbox,
+                    revision_name_textbox,
+                    private,
+                    compute_type,
+                ],
+                submission_result,
+            )
 
-        with gr.TabItem("🚀 Submit", id=5):
             with gr.Column():
-                gr.Markdown(EVALUATION_QUEUE_TEXT)
-                model_name_textbox = gr.Textbox(label="Model name")
-                revision_name_textbox = gr.Textbox(label="Revision", value="main")
-                compute_type = gr.Dropdown(choices=[i.value.name for i in ComputeDtype if i.value.name != "All"], label="Compute dtype", value="float16")
-                submit_button = gr.Button("Submit Eval")
-                submission_result = gr.Markdown()
-                submit_button.click(add_new_eval, [model_name_textbox, revision_name_textbox, gr.State(False), compute_type], submission_result)
-
-            with gr.Accordion(f"✅ Finished Evaluations", open=False):
-                # 修复对齐
-                q_cols = [c for c in EVAL_COLS if c in finished_eval_queue_df.columns]
-                if not q_cols: q_cols = list(finished_eval_queue_df.columns)
-                gr.Dataframe(value=finished_eval_queue_df[q_cols], headers=q_cols, datatype=EVAL_TYPES, column_count=(len(q_cols), "fixed"))
+                with gr.Accordion(
+                    f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                    open=False,
+                ):
+                    with gr.Row():
+                        finished_eval_table = gr.components.Dataframe(
+                            value=finished_eval_queue_df,
+                            headers=EVAL_COLS,
+                            datatype=EVAL_TYPES,
+                            row_count=5,
+                        )
+                with gr.Accordion(
+                    f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                    open=False,
+                ):
+                    with gr.Row():
+                        running_eval_table = gr.components.Dataframe(
+                            value=running_eval_queue_df,
+                            headers=EVAL_COLS,
+                            datatype=EVAL_TYPES,
+                            row_count=5,
+                        )
+
+                with gr.Accordion(
+                    f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                    open=False,
+                ):
+                    with gr.Row():
+                        pending_eval_table = gr.components.Dataframe(
+                            value=pending_eval_queue_df,
+                            headers=EVAL_COLS,
+                            datatype=EVAL_TYPES,
+                            row_count=5,
+                        )
 
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
-            gr.Textbox(value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=10, show_copy_button=True)
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                lines=20,
+                elem_id="citation-button",
+                buttons=["copy"],
+            )
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", hours=3)
+scheduler.add_job(restart_space, "interval", hours=3) # restarted every 3h
+scheduler.add_job(update_dynamic_files, "interval", hours=12) # launched every 2 hour
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+
+demo.queue(default_concurrency_limit=40).launch(css=custom_css)

requirements.txt

@@ -1,20 +1,20 @@
-APScheduler==3.10.1
-black==23.11.0
-click==8.1.3
-datasets==2.14.5
-huggingface-hub>=0.18.0,<1.0.0
-matplotlib==3.7.1
-numpy==1.24.2
-pandas==2.0.0
-plotly==5.14.1
-python-dateutil==2.8.2
-requests==2.28.2
-sentencepiece
-tqdm==4.65.0
-transformers==4.39.0
-tokenizers>=0.15.0
-#gradio-space-ci @ git+https://huggingface.co/spaces/Wauplin/gradio-space-ci@0.2.1 # CI !!!
-gradio==3.28.0
-GitPython==3.1.40
-pydantic==1.10.15
-plotly==5.14.1
+pandas
+numpy
+matplotlib
+plotly
+apscheduler
+tqdm
+requests
+python-dateutil
+
+
+huggingface-hub
+
+transformers
+gradio==6.5.1
+
+datasets
+tokenizers
+GitPython
+
+pydantic>=2.0

src/display/css_html_js.py

@@ -13,9 +13,112 @@ table th:first-child {
     white-space: nowrap;
 }
 
+.custom-modal:not([style*="display: none"]):not(.hidden) {
+    position: fixed !important;
+    top: 0 !important;
+    left: 0 !important;
+    width: 100vw !important;
+    height: 100vh !important;
+    background-color: rgba(0, 0, 0, 0.85) !important;
+    z-index: 10000 !important;
+    display: block !important; 
+    overflow-y: auto !important; 
+    pointer-events: auto !important;
+    padding: 5vh 0 !important; 
+}
+
+.custom-modal[style*="display: none"], 
+.custom-modal.hidden,
+.modal-hidden {
+    display: none !important;
+    visibility: hidden !important;
+    pointer-events: none !important;
+    position: absolute !important; 
+    width: 0 !important;
+    height: 0 !important;
+}
+
+.modal-content {
+    background: white !important;
+    padding: 30px 50px !important;
+    border-radius: 12px;
+    width: 85% !important;
+    max-width: 1100px;
+    margin: 0 auto !important; 
+    display: block !important; 
+    height: auto !important;   
+    min-height: 200px !important;
+    max-height: none !important; 
+    z-index: 10001;
+    text-align: center; 
+    box-shadow: 0 4px 20px rgba(0,0,0,0.3);
+}
+
+.modal-content .gradio-plot, 
+.modal-content .plot-container {
+    height: 400px !important;
+    min-height: 400px !important;
+    max-height: 400px !important;
+    width: 100% !important;
+}
+
+.modal-content .gradio-dataframe {
+    height: auto !important;
+    min-height: 50px !important;
+    max-height: 400px !important;
+    overflow-y: auto !important;
+}
+
+.modal-content .form,
+.modal-content .gap {
+    gap: 0 !important;
+    padding: 0 !important;
+    margin: 0 !important;
+}
+
+.modal-content > * {
+    margin: 0 auto 20px auto !important;
+    flex: none !important;
+    display: block !important;
+}
+
 /* Full width space */
 .gradio-container {
-  max-width: 95%!important;
+    display: flex !important;
+    flex-direction: column !important;
+    align-items: center !important; 
+    width: 95% !important;        
+    max-width: 95% !important;    
+    margin-left: auto !important;  
+    margin-right: auto !important;
+}
+
+.gradio-group {
+    background-color: #fff !important;
+    border: none !important;
+    box-shadow: none !important;
+}
+
+#compare-button-full {
+    height: 100% !important;   
+    width: 100% !important;    
+    display: flex !important;
+    align-items: center !important;
+    justify-content: center !important;
+    min-height: 100px;        
+}
+
+#quant-config-container {
+    border: 1px solid #e5e7eb !important; 
+    border-radius: 8px !important;        
+    background-color: transparent !important; 
+}
+
+.quant-config-header {
+    border-radius: 8px 8px 0 0; 
+    font-weight: 600;
+    background-color: #fff; 
+    color: #71717a; 
 }
 
 /* Text style and margins */
@@ -44,7 +147,7 @@ table th:first-child {
     background: none;
     border: none;
 }
- 
+
 #search-bar {
     padding: 0px;
 }
@@ -71,7 +174,7 @@ table th:first-child {
 
 /* 100% scale*/
 @media (resolution: 96dpi), (min-resolution: 1dppx) and (max-resolution: 1.25dppx) {
-   #model_comparison {
+#model_comparison {
         height: 6rem !important;
         overflow: auto !important;
     }
@@ -93,13 +196,10 @@ table th:first-child {
     }
 
     #component-31 {
-      margin-top: 0.5rem !important;
+    margin-top: 0.5rem !important;
     }
 }
 
-
-
-
 #model_comparison {
         height: 6rem !important;
         overflow: auto !important;
@@ -109,8 +209,8 @@ table th:first-child {
     font-size: 0.7rem !important;
 }
 
-.tab-buttons button {
-    font-size: 20px;
+.tab-buttons > div > button {
+    font-size: 18px !important; 
 }
 
 /* Filters style */
@@ -147,11 +247,3 @@ table th:first-child {
     border: 0
 }
 """
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

src/display/utils.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass, make_dataclass
+from dataclasses import dataclass, make_dataclass, field
 from enum import Enum
 
 import pandas as pd
@@ -44,39 +44,74 @@ class ColumnContent:
 
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append([
+    "model_type_symbol", 
+    ColumnContent, 
+    field(default_factory=lambda: ColumnContent("T", "Type", "str"))
+])
+
+auto_eval_column_dict.append([
+    "model", 
+    ColumnContent, 
+    field(default_factory=lambda: ColumnContent("Model", "markdown", True, never_hidden=True))
+])
+
+# Scores
+auto_eval_column_dict.append([
+    "average", 
+    ColumnContent, 
+    field(default_factory=lambda: ColumnContent("Average ⬆️", "number", True))
+])
+
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", True)])
-auto_eval_column_dict.append(["model_size", ColumnContent, ColumnContent("#Size (G)", "number", True)])
-# Dummy column for the search bar (hidden by the custom CSS)
-auto_eval_column_dict.append(["dummy", ColumnContent, ColumnContent("model_name_for_query", "str", False, dummy=True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False, hidden=True)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["quant_type", ColumnContent, ColumnContent("Quant type", "str", False)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["weight_dtype", ColumnContent, ColumnContent("Weight dtype", "str", False)])
-auto_eval_column_dict.append(["compute_dtype", ColumnContent, ColumnContent("Compute dtype", "str", False)])
-auto_eval_column_dict.append(["merged", ColumnContent, ColumnContent("Merged", "bool", False, hidden=True)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False, hidden=True)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-auto_eval_column_dict.append(["flagged", ColumnContent, ColumnContent("Flagged", "bool", False, hidden=True)])
-auto_eval_column_dict.append(["moe", ColumnContent, ColumnContent("MoE", "bool", False, hidden=True)])
-auto_eval_column_dict.append(["double_quant", ColumnContent, ColumnContent("Double Quant", "bool", False)])
-auto_eval_column_dict.append(["group_size", ColumnContent, ColumnContent("Group Size", "bool", False)])
+    auto_eval_column_dict.append([
+        task.name, 
+        ColumnContent, 
+        field(default_factory=lambda t=task: ColumnContent(t.value.col_name, "number", True))
+    ])
+
+auto_eval_column_dict.append([
+    "params", 
+    ColumnContent, 
+    field(default_factory=lambda: ColumnContent("#Params (B)", "number", True))
+])
+
+auto_eval_column_dict.append([
+    "model_size", 
+    ColumnContent, 
+    field(default_factory=lambda: ColumnContent("#Size (G)", "number", True))
+])
+
+# Dummy column for the search bar
+auto_eval_column_dict.append([
+    "dummy", 
+    ColumnContent, 
+    field(default_factory=lambda: ColumnContent("model_name_for_query", "str", False, dummy=True))
+])
+
+auto_eval_column_dict.append(["model_type", ColumnContent, field(default_factory=lambda: ColumnContent("Type", "str", False, hidden=True))])
+auto_eval_column_dict.append(["architecture", ColumnContent, field(default_factory=lambda: ColumnContent("Architecture", "str", False))])
+auto_eval_column_dict.append(["weight_type", ColumnContent, field(default_factory=lambda: ColumnContent("Weight type", "str", False, True))])
+auto_eval_column_dict.append(["quant_type", ColumnContent, field(default_factory=lambda: ColumnContent("Quant type", "str", False))])
+auto_eval_column_dict.append(["precision", ColumnContent, field(default_factory=lambda: ColumnContent("Precision", "str", False))])
+auto_eval_column_dict.append(["weight_dtype", ColumnContent, field(default_factory=lambda: ColumnContent("Weight dtype", "str", False))])
+auto_eval_column_dict.append(["compute_dtype", ColumnContent, field(default_factory=lambda: ColumnContent("Compute dtype", "str", False))])
+auto_eval_column_dict.append(["merged", ColumnContent, field(default_factory=lambda: ColumnContent("Merged", "bool", False, hidden=True))])
+auto_eval_column_dict.append(["license", ColumnContent, field(default_factory=lambda: ColumnContent("Hub License", "str", False))])
+auto_eval_column_dict.append(["likes", ColumnContent, field(default_factory=lambda: ColumnContent("Hub ❤️", "number", False))])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, field(default_factory=lambda: ColumnContent("Available on the hub", "bool", False, hidden=True))])
+auto_eval_column_dict.append(["revision", ColumnContent, field(default_factory=lambda: ColumnContent("Model sha", "str", False, False))])
+auto_eval_column_dict.append(["flagged", ColumnContent, field(default_factory=lambda: ColumnContent("Flagged", "bool", False, hidden=True))])
+auto_eval_column_dict.append(["moe", ColumnContent, field(default_factory=lambda: ColumnContent("MoE", "bool", False, hidden=True))])
+auto_eval_column_dict.append(["double_quant", ColumnContent, field(default_factory=lambda: ColumnContent("Double Quant", "bool", False))])
+auto_eval_column_dict.append(["group_size", ColumnContent, field(default_factory=lambda: ColumnContent("Group Size", "bool", False))])
 # We use make dataclass to dynamically fill the scores from Tasks
 # auto_eval_column_dict.sort(key=lambda x: x[0])
 sorted_columns = sorted(auto_eval_column_dict[3:], key=lambda x: x[0])
 sorted_auto_eval_column_dict = auto_eval_column_dict[:3] + sorted_columns
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+AutoEvalColumn = make_dataclass("AutoEvalColumn", sorted_auto_eval_column_dict, frozen=True)
+auto_eval_cols = AutoEvalColumn()
+
 
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -87,31 +122,28 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
+eval_queue_cols = EvalQueueColumn()
 
 baseline_row = {
-    AutoEvalColumn.model.name: "<p>Baseline</p>",
-    AutoEvalColumn.revision.name: "N/A",
-    AutoEvalColumn.precision.name: None,
-    AutoEvalColumn.merged.name: False,
-    AutoEvalColumn.average.name: 31.0,
-    AutoEvalColumn.arc.name: 25.0,
-    # AutoEvalColumn.hellaswag.name: 25.0,
-    # AutoEvalColumn.truthfulqa.name: 25.0,
-    AutoEvalColumn.winogrande.name: 50.0,
-    # AutoEvalColumn.gsm8k.name: 0.21,
-    AutoEvalColumn.dummy.name: "baseline",
-    AutoEvalColumn.model_type.name: "",
-    AutoEvalColumn.flagged.name: False,
-    # low-bite new params
-    AutoEvalColumn.mmlu.name: 25.0,
-    AutoEvalColumn.lambada_openai.name: 25.0,
-    AutoEvalColumn.hellaswag.name: 25.0,
-    AutoEvalColumn.piqa.name: 25.0,
-    AutoEvalColumn.truthfulqa_mc.name: 25.0,
-    AutoEvalColumn.openbookqa.name: 25.0,
-    AutoEvalColumn.boolq.name: True,
-    AutoEvalColumn.arc_easy.name: 25.0,
-    AutoEvalColumn.double_quant.name: False,
+    auto_eval_cols.model.name: "<p>Baseline</p>",
+    auto_eval_cols.revision.name: "N/A",
+    auto_eval_cols.precision.name: None,
+    auto_eval_cols.merged.name: False,
+    auto_eval_cols.average.name: 31.0,
+    auto_eval_cols.arc.name: 25.0,
+    auto_eval_cols.winogrande.name: 50.0,
+    auto_eval_cols.dummy.name: "baseline",
+    auto_eval_cols.model_type.name: "",
+    auto_eval_cols.flagged.name: False,
+    auto_eval_cols.mmlu.name: 25.0,
+    auto_eval_cols.lambada_openai.name: 25.0,
+    auto_eval_cols.hellaswag.name: 25.0,
+    auto_eval_cols.piqa.name: 25.0,
+    auto_eval_cols.truthfulqa_mc.name: 25.0,
+    auto_eval_cols.openbookqa.name: 25.0,
+    auto_eval_cols.boolq.name: True,
+    auto_eval_cols.arc_easy.name: 25.0,
+    auto_eval_cols.double_quant.name: False,
 }
 
 # Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)
@@ -123,20 +155,16 @@ baseline_row = {
 # GSM8K: paper
 # Define the human baselines
 human_baseline_row = {
-    AutoEvalColumn.model.name: "<p>Human performance</p>",
-    AutoEvalColumn.revision.name: "N/A",
-    AutoEvalColumn.precision.name: None,
-    AutoEvalColumn.average.name: 92.75,
-    AutoEvalColumn.merged.name: False,
-    AutoEvalColumn.arc.name: 80.0,
-    # AutoEvalColumn.hellaswag.name: 95.0,
-    # AutoEvalColumn.mmlu.name: 89.8,
-    # AutoEvalColumn.truthfulqa.name: 94.0,
-    AutoEvalColumn.winogrande.name: 94.0,
-    # AutoEvalColumn.gsm8k.name: 100,
-    AutoEvalColumn.dummy.name: "human_baseline",
-    AutoEvalColumn.model_type.name: "",
-    AutoEvalColumn.flagged.name: False,
+    auto_eval_cols.model.name: "<p>Human performance</p>",
+    auto_eval_cols.revision.name: "N/A",
+    auto_eval_cols.precision.name: None,
+    auto_eval_cols.average.name: 92.75,
+    auto_eval_cols.merged.name: False,
+    auto_eval_cols.arc.name: 80.0,
+    auto_eval_cols.winogrande.name: 94.0,
+    auto_eval_cols.dummy.name: "human_baseline",
+    auto_eval_cols.model_type.name: "",
+    auto_eval_cols.flagged.name: False,
 }
 
 @dataclass
@@ -355,8 +383,8 @@ class Precision(Enum):
 
 
 # Column selection
-COLS = [c.name for c in fields(AutoEvalColumn)]
-TYPES = [c.type for c in fields(AutoEvalColumn)]
+COLS = [c.name for c in fields(auto_eval_cols)]
+TYPES = [c.type for c in fields(auto_eval_cols)]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
@@ -383,4 +411,4 @@ NUMERIC_MODELSIZE = {
     "~48": pd.Interval(36, 48, closed="right"),
     "~64": pd.Interval(48, 64, closed="right"),
     ">72": pd.Interval(64, 200, closed="right"),
-}
+}

src/leaderboard/filter_models.py

@@ -1,5 +1,5 @@
 from src.display.formatting import model_hyperlink
-from src.display.utils import AutoEvalColumn
+from src.display.utils import auto_eval_cols
 
 # Models which have been flagged by users as being problematic for a reason or another
 # (Model name to forum discussion link)
@@ -130,8 +130,9 @@ DO_NOT_SUBMIT_MODELS = [
 
 def flag_models(leaderboard_data: list[dict]):
     for model_data in leaderboard_data:
+        # 修改点 2：将 AutoEvalColumn 替换为 auto_eval_cols
         # Merges and moes are flagged automatically
-        if model_data[AutoEvalColumn.flagged.name] == True:
+        if model_data.get(auto_eval_cols.flagged.name) == True:
             flag_key = "merged"
         else:
             flag_key = model_data["model_name_for_query"]
@@ -143,11 +144,11 @@ def flag_models(leaderboard_data: list[dict]):
                 f"See discussion #{issue_num}",
             )
             model_data[
-                AutoEvalColumn.model.name
-            ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
-            model_data[AutoEvalColumn.flagged.name] = True
+                auto_eval_cols.model.name
+            ] = f"{model_data[auto_eval_cols.model.name]} has been flagged! {issue_link}"
+            model_data[auto_eval_cols.flagged.name] = True
         else:
-            model_data[AutoEvalColumn.flagged.name] = False
+            model_data[auto_eval_cols.flagged.name] = False
 
 
 def remove_forbidden_models(leaderboard_data: list[dict]):

src/leaderboard/read_evals.py

@@ -11,7 +11,7 @@ import numpy as np
 from huggingface_hub import ModelCard
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, QuantType, WeightDtype, ComputeDtype 
+from src.display.utils import auto_eval_cols, ModelType, Tasks, Precision, WeightType, QuantType, WeightDtype, ComputeDtype 
 
 
 @dataclass
@@ -60,10 +60,10 @@ class EvalResult:
         quant_type = QuantType.from_str(str(config.get("quant_type", "GPTQ")))
         weight_dtype = WeightDtype.from_str(data["task_info"].get("weight_dtype", "int4"))
         compute_dtype = ComputeDtype.from_str(data["task_info"].get("compute_dtype", "bfloat16"))
-        # double_quant = data["quantization_config"].get("bnb_4bit_use_double_quant", False)
+        
         model_params = round(float(config["model_params"]), 2)
         model_size = round(float(config["model_size"]), 2)
-        # group_size = data["quantization_config"].get("group_size", -1)
+        
         if data.get("quantization_config", None):
             double_quant = data["quantization_config"].get("bnb_4bit_use_double_quant", False)
             group_size = data["quantization_config"].get("group_size", -1)
@@ -81,7 +81,6 @@ class EvalResult:
 
         if local and org_and_model[0] != "Intel":
             org_and_model = config.get("model_name").split("/")
-            # temporary "local"
             org_and_model = ["local", org_and_model[-1]]
             quant_type = QuantType.autoround
 
@@ -95,7 +94,7 @@ class EvalResult:
             result_key = f"{org}_{model}_{precision.value.name}"
         full_model = "/".join(org_and_model)
 
-        # Extract results available in this file (some results are split in several files)
+        # Extract results
         results = {}
         for task in Tasks:
             task = task.value
@@ -137,19 +136,12 @@ class EvalResult:
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
-            # self.model_type = ModelType.from_str(request.get("model_type", "Unknown"))
-            # self.precision = WeightType[request.get("weight_type", "Original")]
-            # self.num_params = request.get("model_size", 0) / 2 # need fix
             self.date = request.get("submitted_time", "")
             self.architecture = request.get("architectures", "Unknown")
             self.status = request.get("status", "Failed")
         except Exception as e:
-            print(requests_path, self.full_model,
-                self.quant_type.value.name, self.precision.value.name,
-                self.weight_dtype.value.name, self.compute_dtype.value.name)
             self.status = "Failed"
             print(f"Could not find request file for {self.org}/{self.model}")
-            print(traceback.format_exc())
 
     def update_with_dynamic_file_dict(self, file_dict):
         self.license = file_dict.get("license", "?")
@@ -161,57 +153,67 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-
+        
+        valid_results = [v for v in self.results.values() if v is not None]
+        average = sum(valid_results) / len(Tasks) if len(Tasks) > 0 else 0
+        
         data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.quant_type.name: self.quant_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.quant_type.value.symbol,
-            AutoEvalColumn.weight_dtype.name: self.weight_dtype.value.name,
-            AutoEvalColumn.compute_dtype.name: self.compute_dtype.value.name,
-            AutoEvalColumn.double_quant.name: self.double_quant,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model, self.result_file),
-            AutoEvalColumn.dummy.name: self.full_model,
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.model_size.name: self.model_size,
-            AutoEvalColumn.group_size.name: self.group_size,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-            AutoEvalColumn.merged.name: "merge" in self.tags if self.tags else False,
-            AutoEvalColumn.moe.name: ("moe" in self.tags if self.tags else False) or "moe" in self.full_model.lower(),
-            AutoEvalColumn.flagged.name: self.flagged
+            "eval_name": self.eval_name,
+            "date": self.date, 
+            auto_eval_cols.precision.name: self.precision.value.name,
+            auto_eval_cols.quant_type.name: self.quant_type.value.name,
+            auto_eval_cols.model_type_symbol.name: self.quant_type.value.symbol,
+            auto_eval_cols.weight_dtype.name: self.weight_dtype.value.name,
+            auto_eval_cols.compute_dtype.name: self.compute_dtype.value.name,
+            auto_eval_cols.model.name: make_clickable_model(self.full_model, self.result_file),
+            auto_eval_cols.revision.name: self.revision,
+            auto_eval_cols.average.name: average,
+            auto_eval_cols.model_size.name: self.model_size,
+            auto_eval_cols.dummy.name: self.full_model,
         }
 
+        data_dict[auto_eval_cols.still_on_hub.name] = self.still_on_hub
+        data_dict[auto_eval_cols.flagged.name] = self.flagged
+        
+        if hasattr(auto_eval_cols, "double_quant"):
+            data_dict[auto_eval_cols.double_quant.name] = self.double_quant
+        if hasattr(auto_eval_cols, "architecture"):
+            data_dict[auto_eval_cols.architecture.name] = self.architecture
+        if hasattr(auto_eval_cols, "params"):
+            data_dict[auto_eval_cols.params.name] = self.num_params
+        if hasattr(auto_eval_cols, "license"):
+            data_dict[auto_eval_cols.license.name] = self.license
+        if hasattr(auto_eval_cols, "likes"):
+            data_dict[auto_eval_cols.likes.name] = self.likes
+        if hasattr(auto_eval_cols, "group_size"):
+            data_dict[auto_eval_cols.group_size.name] = self.group_size
+        
+        if hasattr(auto_eval_cols, "merged"):
+            data_dict[auto_eval_cols.merged.name] = "merge" in (self.tags if self.tags else [])
+        if hasattr(auto_eval_cols, "moe"):
+            data_dict[auto_eval_cols.moe.name] = ("moe" in (self.tags if self.tags else [])) or "moe" in self.full_model.lower()
+
         for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+            data_dict[task.value.col_name] = self.results.get(task.value.benchmark, 0)
 
         return data_dict
 
 
+
 def get_request_file_for_model(requests_path, model_name,
         quant_type, precision, weight_dtype, compute_dtype):
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    # {model_path}_eval_request_{private}_{quant_type}_{precision}_{weight_dtype}_{compute_dtype}.json
     request_files = os.path.join(
         requests_path,
         f"{model_name}_eval_request_*.json",
     )
     request_files = glob.glob(request_files)
 
-    # Select correct request file (precision)
     request_file = ""
     request_files = sorted(request_files, reverse=True)
     for tmp_request_file in request_files:
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
-            print(model_name, req_content["precision"], precision.split(".")[-1], str(req_content["quant_type"]), quant_type, req_content["weight_dtype"], weight_dtype.split(".")[-1],req_content["compute_dtype"], compute_dtype.split(".")[-1] )
             if (
                 req_content["status"] in ["Finished"]
                 and req_content["precision"] == precision.split(".")[-1]
@@ -236,48 +238,48 @@ def get_raw_eval_results(results_path: str, requests_path: str, dynamic_path: st
     model_result_filepaths = []
 
     for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue
 
-        # Sort the files by date
         try:
             files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
+        except Exception:
             files = [files[-1]]
 
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
-    with open(dynamic_path) as f:
-        dynamic_data = json.load(f)
+    dynamic_data = {}
+    if os.path.exists(dynamic_path):
+        with open(dynamic_path) as f:
+            dynamic_data = json.load(f)
 
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
-        # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        
         eval_result.update_with_request_file(requests_path)
+        
         if eval_result.full_model in dynamic_data:
-            # eval_result.update_with_dynamic_file_dict(dynamic_data[eval_result.full_model])
-            # Hardcoding because of gating problem
+            eval_result.update_with_dynamic_file_dict(dynamic_data[eval_result.full_model])
             if "meta-llama" in eval_result.full_model: 
                 eval_result.still_on_hub = True
 
-        # Store results of same eval together
         eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
+        if eval_name in eval_results:
             eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
         else:
             eval_results[eval_name] = eval_result
 
-
     results = []
     for v in eval_results.values():
         try:
             if v.status == "Finished":
-                v.to_dict() # we test if the dict version is complete
+                v.to_dict() 
                 results.append(v)
-        except KeyError:  # not all eval values present
+        except Exception as e:
+            print(f"Error processing {v.eval_name}: {e}")
             continue
 
     return results
+

src/populate.py

@@ -4,7 +4,7 @@ import os
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn, baseline_row
+from src.display.utils import auto_eval_cols, eval_queue_cols, baseline_row
 from src.leaderboard.filter_models import filter_models_flags
 from src.leaderboard.read_evals import get_raw_eval_results
 
@@ -12,20 +12,23 @@ from src.leaderboard.read_evals import get_raw_eval_results
 def get_leaderboard_df(results_path: str, requests_path: str, dynamic_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     raw_data = get_raw_eval_results(results_path=results_path, requests_path=requests_path, dynamic_path=dynamic_path)
     all_data_json = [v.to_dict() for v in raw_data]
-    print(all_data_json)
+    
     all_data_json.append(baseline_row)
     filter_models_flags(all_data_json)
-    print("Keys in the first record of all_data_json:", all_data_json[0].keys())
-
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    print("Columns used in DataFrame:", cols, df.columns)
-    df = df[cols].round(decimals=2)
-
+    
+    avg_col = auto_eval_cols.average.name
+    if avg_col in df.columns:
+        df = df.sort_values(by=[avg_col], ascending=False)
+    
+    existing_cols = [c for c in cols if c in df.columns]
+    df = df[existing_cols].round(decimals=2)
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
+    existing_benchmarks = [c for c in benchmark_cols if c in df.columns]
+    if existing_benchmarks:
+        df = df[has_no_nan_values(df, existing_benchmarks)]
+    
     return raw_data, df
 
 
@@ -39,8 +42,8 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             with open(file_path) as fp:
                 data = json.load(fp)
 
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            data[eval_queue_cols.model.name] = make_clickable_model(data["model"])
+            data[eval_queue_cols.revision.name] = data.get("revision", "main")
 
             all_evals.append(data)
         elif ".md" not in entry:
@@ -51,14 +54,18 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
                 with open(file_path) as fp:
                     data = json.load(fp)
 
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                data[eval_queue_cols.model.name] = make_clickable_model(data["model"])
+                data[eval_queue_cols.revision.name] = data.get("revision", "main")
                 all_evals.append(data)
 
     pending_list = [e for e in all_evals if e["status"] in ["Pending", "Rerun", "Waiting"]]
     running_list = [e for e in all_evals if e["status"] == "Running"]
     finished_list = [e for e in all_evals if e["status"].startswith("Finished") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]
+    
+    existing_q_cols = [c for c in cols if c in pd.DataFrame(all_evals).columns] if all_evals else cols
+
+    df_pending = pd.DataFrame.from_records(pending_list, columns=existing_q_cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=existing_q_cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=existing_q_cols)
+    
+    return df_finished[existing_q_cols], df_running[existing_q_cols], df_pending[existing_q_cols]

src/tools/plots.py

@@ -4,43 +4,44 @@ import plotly.express as px
 from plotly.graph_objs import Figure
 
 from src.leaderboard.filter_models import FLAGGED_MODELS
-from src.display.utils import human_baseline_row as HUMAN_BASELINE, AutoEvalColumn, Tasks, Task, BENCHMARK_COLS
+from src.display.utils import human_baseline_row as HUMAN_BASELINE, auto_eval_cols, Tasks, Task, BENCHMARK_COLS
 from src.leaderboard.read_evals import EvalResult
 
 
-
 def create_scores_df(raw_data: list[EvalResult]) -> pd.DataFrame:
     """
     Generates a DataFrame containing the maximum scores until each date.
-
-    :param results_df: A DataFrame containing result information including metric scores and dates.
-    :return: A new DataFrame containing the maximum scores until each date for every metric.
     """
-    # Step 1: Ensure 'date' is in datetime format and sort the DataFrame by it
-    results_df = pd.DataFrame(raw_data)
-    #results_df["date"] = pd.to_datetime(results_df["date"], format="mixed", utc=True)
-    results_df.sort_values(by="date", inplace=True)
+    all_data = [v.to_dict() for v in raw_data]
+    results_df = pd.DataFrame(all_data)
+    
+    if "date" in results_df.columns:
+        results_df.sort_values(by="date", inplace=True)
 
-    # Step 2: Initialize the scores dictionary
-    scores = {k: [] for k in BENCHMARK_COLS + [AutoEvalColumn.average.name]}
+    scores = {k: [] for k in BENCHMARK_COLS + [auto_eval_cols.average.name]}
 
-    # Step 3: Iterate over the rows of the DataFrame and update the scores dictionary
-    for task in [t.value for t in Tasks] + [Task("Average", "avg", AutoEvalColumn.average.name)]:
+    for task in [t.value for t in Tasks] + [Task("Average", "avg", auto_eval_cols.average.name)]:
         current_max = 0
         last_date = ""
         column = task.col_name
+        
         for _, row in results_df.iterrows():
-            current_model = row["full_model"]
-            # We ignore models that are flagged/no longer on the hub/not finished 
-            to_ignore = not row["still_on_hub"] or row["flagged"] or current_model in FLAGGED_MODELS or row["status"] != "Finished"
+            current_model = row.get("dummy", "Unknown") 
+            
+            still_on_hub = row.get(auto_eval_cols.still_on_hub.name, True)
+            is_flagged = row.get(auto_eval_cols.flagged.name, False)
+            
+            to_ignore = not still_on_hub or is_flagged or current_model in FLAGGED_MODELS
             if to_ignore:
                 continue
 
-            current_date = row["date"]
+            current_date = row.get("date", "")
+            if not current_date: continue
+
             if task.benchmark == "Average":
-                current_score = np.mean(list(row["results"].values()))
+                current_score = row.get(auto_eval_cols.average.name, 0)
             else:
-                current_score = row["results"][task.benchmark]
+                current_score = row.get(task.col_name, 0)
 
             if current_score > current_max:
                 if current_date == last_date and len(scores[column]) > 0:
@@ -50,57 +51,36 @@ def create_scores_df(raw_data: list[EvalResult]) -> pd.DataFrame:
                 current_max = current_score
                 last_date = current_date
 
-    # Step 4: Return all dictionaries as DataFrames
     return {k: pd.DataFrame(v) for k, v in scores.items()}
 
 
-def create_plot_df(scores_df: dict[str: pd.DataFrame]) -> pd.DataFrame:
-    """
-    Transforms the scores DataFrame into a new format suitable for plotting.
-
-    :param scores_df: A DataFrame containing metric scores and dates.
-    :return: A new DataFrame reshaped for plotting purposes.
-    """
-    # Initialize the list to store DataFrames
+def create_plot_df(scores_df: dict[str, pd.DataFrame]) -> pd.DataFrame:
+    """Reshapes the scores DataFrame for plotting."""
     dfs = []
 
-    # Iterate over the cols and create a new DataFrame for each column
-    for col in BENCHMARK_COLS + [AutoEvalColumn.average.name]:
-        d = scores_df[col].reset_index(drop=True)
-        d["task"] = col
-        dfs.append(d)
+    for col in BENCHMARK_COLS + [auto_eval_cols.average.name]:
+        if col in scores_df and not scores_df[col].empty:
+            d = scores_df[col].reset_index(drop=True)
+            d["task"] = col
+            dfs.append(d)
 
-    # Concatenate all the created DataFrames
-    concat_df = pd.concat(dfs, ignore_index=True)
+    if not dfs:
+        return pd.DataFrame(columns=["model", "date", "score", "task"])
 
-    # Sort values by 'date'
+    concat_df = pd.concat(dfs, ignore_index=True)
     concat_df.sort_values(by="date", inplace=True)
     concat_df.reset_index(drop=True, inplace=True)
     return concat_df
 
 
-def create_metric_plot_obj(
-    df: pd.DataFrame, metrics: list[str], title: str
-) -> Figure:
-    """
-    Create a Plotly figure object with lines representing different metrics
-    and horizontal dotted lines representing human baselines.
-
-    :param df: The DataFrame containing the metric values, names, and dates.
-    :param metrics: A list of strings representing the names of the metrics
-                    to be included in the plot.
-    :param title: A string representing the title of the plot.
-    :return: A Plotly figure object with lines representing metrics and
-             horizontal dotted lines representing human baselines.
-    """
+def create_metric_plot_obj(df: pd.DataFrame, metrics: list[str], title: str) -> Figure:
+    if df.empty:
+        return px.line(title="No data available")
 
-    # Filter the DataFrame based on the specified metrics
     df = df[df["task"].isin(metrics)]
 
-    # Filter the human baselines based on the specified metrics
     filtered_human_baselines = {k: v for k, v in HUMAN_BASELINE.items() if k in metrics}
 
-    # Create a line figure using plotly express with specified markers and custom data
     fig = px.line(
         df,
         x="date",
@@ -111,33 +91,21 @@ def create_metric_plot_obj(
         title=title,
     )
 
-    # Update hovertemplate for better hover interaction experience
     fig.update_traces(
-        hovertemplate="<br>".join(
-            [
-                "Model Name: %{customdata[2]}",
-                "Metric Name: %{customdata[0]}",
-                "Date: %{x}",
-                "Metric Value: %{y}",
-            ]
-        )
+        hovertemplate="<br>".join([
+            "Model Name: %{customdata[2]}",
+            "Metric Name: %{customdata[0]}",
+            "Date: %{x}",
+            "Metric Value: %{y}",
+        ])
     )
 
-    # Update the range of the y-axis
     fig.update_layout(yaxis_range=[0, 100])
+    metric_color_mapping = {trace.name: trace.line.color for trace in fig.data}
 
-    # Create a dictionary to hold the color mapping for each metric
-    metric_color_mapping = {}
-
-    # Map each metric name to its color in the figure
-    for trace in fig.data:
-        metric_color_mapping[trace.name] = trace.line.color
-
-    # Iterate over filtered human baselines and add horizontal lines to the figure
     for metric, value in filtered_human_baselines.items():
-        color = metric_color_mapping.get(metric, "blue")  # Retrieve color from mapping; default to blue if not found
-        location = "top left" if metric == "HellaSwag" else "bottom left"  # Set annotation position
-        # Add horizontal line with matched color and positioned annotation
+        color = metric_color_mapping.get(metric, "blue")
+        location = "top left" if metric == "HellaSwag" else "bottom left"
         fig.add_hline(
             y=value,
             line_dash="dot",
@@ -148,9 +116,4 @@ def create_metric_plot_obj(
             line_color=color,
         )
 
-    return fig
-
-
-# Example Usage:
-# human_baselines dictionary is defined.
-# chart = create_metric_plot_obj(scores_df, ["ARC", "HellaSwag", "MMLU", "TruthfulQA"], human_baselines, "Graph Title")
+    return fig