init

app.py

@@ -3,6 +3,7 @@ from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
+from src.data_utils import get_dataframe_category, get_dataframe_language
 
 from src.about import (
     CITATION_BUTTON_LABEL,
@@ -57,37 +58,39 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
+# def init_leaderboard(dataframe):
+#     if dataframe is None or dataframe.empty:
+#         raise ValueError("Leaderboard DataFrame is empty or None.")
+#     return Leaderboard(
+#         value=dataframe,
+#         datatype=[c.type for c in fields(AutoEvalColumn)],
+#         select_columns=SelectColumns(
+#             default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+#             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+#             label="Select Columns to Display:",
+#         ),
+#         search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+#         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+#         filter_columns=[
+#             ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+#             ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+#             ColumnFilter(
+#                 AutoEvalColumn.params.name,
+#                 type="slider",
+#                 min=0.01,
+#                 max=150,
+#                 label="Select the number of parameters (B)",
+#             ),
+#             ColumnFilter(
+#                 AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+#             ),
+#         ],
+#         bool_checkboxgroup_label="Hide models",
+#         interactive=False,
+#     )
+
 
+tab_keys = ["Category", "Language"]
 
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -95,8 +98,91 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+        
+        def search_leaderboard(query, df):
+            if not query.strip():
+                return df
+            filtered = df[df.apply(lambda row: row.astype(str).str.contains(query, case=False).any(), axis=1)]
+            return filtered
+        
+        def update_modelselector_group(groups, df):
+            """
+            groups (gr.CheckboxGroup): List of currently selected models
+            df (DataFrame or gr.State): Current dataframe
+            """
+            print("groups:", groups)
+            if not groups:
+                return None
+            
+            filtered_df = df[df["Group"].isin(groups)]
+            models = filtered_df["Model Name"].unique().tolist()
+
+            return models
+
+        def update_columnselector_group(columns, groups, df):
+            print("column groups:", groups)
+
+            columns = [c for c in columns if c in df.columns[:3]]
+
+            columns.extend(df.columns[3:])
+            
+            print(columns)
+            
+            return columns
+        
+
+        def update_leaderboard(models, columns, df):
+            print("models:", models)
+            print("columns:", columns)
+            
+            filtered_df = df[df["Model Name"].isin(models)]
+            filtered_columns = [c for c in df.columns if c in columns or c in ["Model Name"]]
+            filtered_df = filtered_df[filtered_columns]
+
+            for col in filtered_df.select_dtypes(include="number").columns:
+                filtered_df[col] = filtered_df[col].round(3)
+
+            return filtered_df
+        
+        def get_models_by_group(df, groups):
+            return df[df["Group"].isin(groups)]["Model Name"].tolist()
+        
+        for _, key in enumerate(tab_keys):
+            with gr.TabItem(key, visible=True):
+                if key == "Category":
+                    df = get_dataframe_category()
+                else:
+                    df = get_dataframe_language()
+                df_state = gr.State(df)
+
+                with gr.Row():
+                    with gr.Column():
+                        search_box = gr.Textbox(label="Search Model by Name")
+                        group_list = df["Group"].unique().tolist()
+                        group_selector = gr.CheckboxGroup(choices=df["Group"].unique().tolist(), value=group_list, label="Select Model Group")
+
+                        if key == "Category":
+                            column_selector = gr.CheckboxGroup(choices=df.columns.tolist()[3:], value=configs.ON_LOAD_COLUMNS_CATEGORY[3:], label="Select Columns")
+                        else:
+                            column_selector = gr.CheckboxGroup(choices=df.columns.tolist()[3:], value=configs.ON_LOAD_COLUMNS_LANG[3:], label="Select Columns")
+
+                    with gr.Column():
+                        with gr.Accordion("세부 사항", open=False):
+                            model_group = df["Model Name"].tolist()
+                            model_selector = gr.CheckboxGroup(choices=df["Model Name"].tolist(), value=model_group, label="Select Models")
+
+                ld = gr.DataFrame(
+                    value=df.round(3)
+                )
+
+                # Define change functions for user interaction
+                search_box.change(fn=search_leaderboard, inputs=[search_box, df_state], outputs=ld)
+                group_selector.change(fn=update_modelselector_group, inputs=[group_selector, df_state], outputs=model_selector)
+                model_selector.change(fn=update_leaderboard, inputs=[model_selector, column_selector, df_state], outputs=ld)
+                column_selector.change(fn=update_leaderboard, inputs=[model_selector, column_selector, df_state], outputs=ld)
+                
+        # with gr.TabItem("Docs"):
+        #     gr.Markdown((Path(__file__).parent / "docs.md").read_text())
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
@@ -201,5 +287,4 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-print("test")
 demo.queue(default_concurrency_limit=40).launch()

src/about.py

@@ -21,11 +21,11 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">🥇 ProductivityBench (v1)</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+ProductivityBench is designed to evaluate LLMs for Productivity Assistants which stand for human's job productivity.
 """
 
 # Which evaluations are you running? how can people reproduce what you have?

src/config.py

@@ -0,0 +1,39 @@
+ON_LOAD_COLUMNS_LANG = [
+    "Model Name",
+    "Group",
+    "Overall",
+    "KO",
+    "EN",
+    "JA",
+    "ZH",
+    "PL",
+    "DE",
+    "PT",
+    "ES",
+    "FR",
+    "IT",
+    "RU",
+    "VI"
+]
+
+ON_LOAD_COLUMNS_CATEGORY = [
+    "Model Name",
+    "Group",
+    "Overall",
+    "Content Generation",
+    "Editing",
+    "Data Analysis",
+    "Reasoning",
+    "Samsung Knowledge",
+    "Hallucination",
+    "Safety",
+    "Repeatition",
+    "Summarization",
+    "Translation",
+    "Multi-Turn"
+]
+
+COLUMN_GROUP_LIST = [
+    "Category",
+    "Language"
+]

src/data/export_category_250618.csv

@@ -8,36 +8,4 @@
 "deepseek_r1"	"DeepSeek"	"55.27"	"61.69"	"54.76"	"68.67"	"68.00"	"46.67"	"51.67"	"20.00"	"46.67"	"67.81"	"49.00"	"43.33"
 "deepseek_r1_0528"	"DeepSeek"	"52.60"	"59.09"	"51.19"	"65.33"	"65.00"	"38.33"	"43.33"	"27.50"	"53.33"	"69.18"	"41.33"	"41.67"
 "deepseek_v3"	"DeepSeek"	"56.99"	"62.99"	"58.93"	"58.00"	"59.00"	"36.67"	"41.67"	"25.00"	"40.00"	"72.60"	"60.00"	"46.67"
-"deepseek_v3_0324"	"DeepSeek"	"54.51"	"55.84"	"48.21"	"63.33"	"70.00"	"43.33"	"50.00"	"20.00"	"46.67"	"72.95"	"49.67"	"43.33"
-"gemini-1.5-flash"	"Gemini"	"45.24"	"50.65"	"42.26"	"46.67"	"43.00"	"20.00"	"53.33"	"21.25"	"13.33"	"66.44"	"40.00"	"39.44"
-"gemini-1.5-pro"	"Gemini"	"52.48"	"57.14"	"50.00"	"50.00"	"54.00"	"43.33"	"51.67"	"33.75"	"30.00"	"69.52"	"52.00"	"40.56"
-"gemini-2.0-flash"	"Gemini"	"55.27"	"54.55"	"54.17"	"56.00"	"51.00"	"58.33"	"60.00"	"20.00"	"40.00"	"74.32"	"56.00"	"42.22"
-"gemini-2.5-pro-05-06"	"Gemini"	"63.98"	"62.99"	"61.90"	"70.67"	"72.00"	"48.33"	"73.33"	"23.75"	"43.33"	"78.77"	"66.00"	"52.78"
-"Gemma-2-27B-it"	"Gemma"	"43.14"	"51.95"	"38.10"	"42.67"	"29.00"	"21.67"	"48.33"	"37.50"	"20.00"	"62.33"	"41.00"	"32.78"
-"Gemma-3-1B-it"	"Gemma"	"12.96"	"25.32"	"10.12"	"15.33"	"9.00"	"0.00"	"11.67"	"27.50"	"6.67"	"22.60"	"2.67"	"6.11"
-"Gemma-3-4B-it"	"Gemma"	"29.61"	"40.91"	"28.57"	"30.00"	"20.00"	"13.33"	"20.00"	"28.75"	"10.00"	"51.03"	"22.00"	"16.11"
-"Gemma-3-12B-it"	"Gemma"	"42.50"	"51.30"	"48.81"	"37.33"	"30.00"	"23.33"	"31.67"	"33.75"	"16.67"	"66.44"	"37.33"	"28.33"
-"Gemma-3-27B-it"	"Gemma"	"44.09"	"53.25"	"44.64"	"50.00"	"39.00"	"33.33"	"45.00"	"26.25"	"23.33"	"63.36"	"33.67"	"34.44"
-"gpt-4o"	"GPT"	"56.42"	"61.04"	"61.31"	"58.67"	"49.00"	"45.00"	"51.67"	"35.00"	"43.33"	"73.29"	"53.00"	"45.56"
-"gpt-o1"	"GPT"	"67.92"	"68.18"	"76.19"	"74.00"	"69.00"	"35.00"	"65.00"	"30.00"	"66.67"	"84.59"	"66.67"	"58.33"
-"gpt-o3"	"GPT"	"70.33"	"76.62"	"75.00"	"74.67"	"79.00"	"53.33"	"58.33"	"23.75"	"76.67"	"83.56"	"74.00"	"53.89"
-"gpt-o4-mini"	"GPT"	"65.31"	"75.97"	"63.69"	"76.00"	"77.00"	"41.67"	"55.00"	"30.00"	"66.67"	"81.85"	"59.67"	"51.67"
-"llama3_1_8b_inst"	"Llama"	"25.79"	"37.66"	"25.00"	"31.33"	"18.00"	"13.33"	"36.67"	"23.75"	"13.33"	"37.67"	"17.00"	"15.00"
-"llama3_1_70b_inst"	"Llama"	"40.79"	"45.45"	"41.67"	"49.33"	"35.00"	"23.33"	"43.33"	"21.25"	"20.00"	"54.79"	"37.33"	"32.22"
-"llama3_1_405b_fp8_inst"	"Llama"	"48.03"	"50.00"	"48.81"	"52.67"	"47.00"	"30.00"	"50.00"	"22.50"	"33.33"	"64.04"	"47.33"	"36.67"
-"llama3_3_70b_inst"	"Llama"	"40.60"	"48.70"	"43.45"	"45.33"	"38.00"	"16.67"	"40.00"	"20.00"	"16.67"	"58.56"	"32.67"	"33.89"
-"llama4_scout"	"Llama"	"44.98"	"46.75"	"39.88"	"52.67"	"43.00"	"31.67"	"41.67"	"22.50"	"23.33"	"61.30"	"44.00"	"37.22"
-"llama4_maverick"	"Llama"	"51.65"	"54.55"	"43.45"	"58.67"	"55.00"	"36.67"	"55.00"	"32.50"	"16.67"	"64.04"	"53.33"	"44.44"
-"Mixtral-8x7B-Instruct-v0.1"	"Mistral"	"22.81"	"26.62"	"16.07"	"24.67"	"13.00"	"16.67"	"38.33"	"23.75"	"23.33"	"37.67"	"13.00"	"18.33"
-"phi-4"	"Phi"	"39.83"	"45.45"	"39.88"	"47.33"	"45.00"	"16.67"	"33.33"	"46.25"	"23.33"	"51.71"	"33.00"	"27.78"
-"Qwen2-72B-Instruct"	"Qwen"	"39.52"	"42.86"	"38.69"	"34.67"	"31.00"	"18.33"	"51.67"	"32.50"	"23.33"	"56.16"	"37.33"	"31.67"
-"Qwen2.5-14B-Instruct"	"Qwen"	"37.99"	"45.45"	"27.98"	"36.67"	"39.00"	"21.67"	"51.67"	"32.50"	"26.67"	"54.45"	"32.67"	"28.89"
-"Qwen2.5-32B-Instruct"	"Qwen"	"43.84"	"51.95"	"38.10"	"47.33"	"45.00"	"21.67"	"55.00"	"35.00"	"20.00"	"63.36"	"36.00"	"31.67"
-"Qwen2.5-72B-Instruct"	"Qwen"	"46.19"	"52.60"	"43.45"	"50.67"	"42.00"	"23.33"	"48.33"	"37.50"	"30.00"	"65.41"	"39.00"	"36.11"
-"Qwen-QwQ-32B"	"Qwen"	"47.46"	"54.55"	"45.24"	"65.33"	"66.00"	"25.00"	"36.67"	"21.25"	"26.67"	"65.07"	"39.33"	"29.44"
-"Qwen3-235B-A22B"	"Qwen"	"48.09"	"59.74"	"41.67"	"65.33"	"71.00"	"33.33"	"41.67"	"20.00"	"33.33"	"66.44"	"30.33"	"38.89"
-"Gauss2.2-37B-Instruct-250430"	"Gauss"	"50.70"	"52.60"	"50.60"	"43.33"	"42.00"	"28.33"	"41.67"	"26.25"	"26.67"	"71.23"	"58.00"	"40.00"
-"Gauss2.2-37B-Think-250430"	"Gauss"	"46.00"	"57.14"	"40.48"	"59.33"	"59.00"	"26.67"	"36.67"	"20.00"	"36.67"	"60.62"	"39.67"	"32.78"
-"GaussO-Owl-Ultra-Think-250604"	"Gauss"	"57.05"	"63.64"	"52.98"	"66.00"	"57.00"	"48.33"	"55.00"	"37.50"	"33.33"	"75.00"	"53.67"	"40.56"
-"GaussO-Owl-Ultra-Think-250423"	"Gauss"	"56.10"	"61.04"	"47.62"	"68.00"	"69.00"	"48.33"	"51.67"	"23.75"	"53.33"	"69.86"	"53.00"	"44.44"
-"GaussO-Owl-Ultra-Instruct-250423"	"Gauss"	"58.58"	"64.94"	"55.95"	"63.33"	"69.00"	"41.67"	"53.33"	"25.00"	"36.67"	"73.97"	"60.00"	"44.44"
+"deepseek_v3_0324"	"DeepSeek"	"54.51"	"55.84"	"48.21"	"63.33"	"70.00"	"43.33"	"50.00"	"20.00"	"46.67"	"72.95"	"49.67"	"43.33"