update

app.py

@@ -38,8 +38,8 @@ from src.envs import (
     DATA_VERSION,
     DATA_REPO,
     HARD_RESULT_REPO,
-    ELO_REPO,
-    HARD_ELO_REPO,
+    # ELO_REPO,  # Comment out
+    # HARD_ELO_REPO,  # Comment out
     SOLVE_REPO,
     HARD_SOLVE_REPO,
     HF_TOKEN,
@@ -51,7 +51,7 @@ from src.envs import (
 )
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.execute import generate_command, default_command, stream_logs, find_result_file
-from src.tools.plots import plot_elo_mle, plot_solve_rate
+from src.tools.plots import plot_solve_rate
 # from src.voting.vote_system import VoteManager, run_scheduler
 
 # Configure logging
@@ -66,10 +66,10 @@ DO_FULL_INIT = True # os.getenv("LEADERBOARD_FULL_INIT", "True") == "True"
 NEW_DATA_ON_LEADERBOARD = True
 LEADERBOARD_DF = None
 HARD_LEADERBOARD_DF = None
-ELO_TASK_DF = None
-ELO_BENCH_DF = None
-HARD_ELO_TASK_DF = None
-HARD_ELO_BENCH_DF = None
+# ELO_TASK_DF = None  # Comment out
+# ELO_BENCH_DF = None  # Comment out
+# HARD_ELO_TASK_DF = None  # Comment out
+# HARD_ELO_BENCH_DF = None  # Comment out
 COMPLETE_SOLVE_DF = None
 INSTRUCT_SOLVE_DF = None
 HARD_COMPLETE_SOLVE_DF = None
@@ -154,10 +154,10 @@ def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3, ba
 def get_latest_data_leaderboard(
     leaderboard_initial_df = None,
     hard_leaderboard_initial_df = None,
-    elo_task_df = None,
-    elo_bench_df = None,
-    hard_elo_task_df = None,
-    hard_elo_bench_df = None,
+    # elo_task_df = None,  # Comment out
+    # elo_bench_df = None,  # Comment out
+    # hard_elo_task_df = None,  # Comment out
+    # hard_elo_bench_df = None,  # Comment out
     complete_solve_df = None,
     instruct_solve_df = None,
     hard_complete_solve_df = None,
@@ -166,10 +166,10 @@ def get_latest_data_leaderboard(
     global NEW_DATA_ON_LEADERBOARD
     global LEADERBOARD_DF
     global HARD_LEADERBOARD_DF
-    global ELO_TASK_DF
-    global ELO_BENCH_DF
-    global HARD_ELO_TASK_DF
-    global HARD_ELO_BENCH_DF
+    # global ELO_TASK_DF  # Comment out
+    # global ELO_BENCH_DF  # Comment out
+    # global HARD_ELO_TASK_DF  # Comment out
+    # global HARD_ELO_BENCH_DF  # Comment out
     global COMPLETE_SOLVE_DF
     global INSTRUCT_SOLVE_DF
     global HARD_COMPLETE_SOLVE_DF
@@ -182,7 +182,7 @@ def get_latest_data_leaderboard(
             "default", 
             split="train", 
             cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS,
             verification_mode="no_checks"
         ).filter(lambda x: x['complete'] is not None or x['instruct'] is not None)
         LEADERBOARD_DF = get_leaderboard_df(
@@ -194,7 +194,7 @@ def get_latest_data_leaderboard(
             "default", 
             split="train", 
             cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS,
             verification_mode="no_checks"
         ).filter(lambda x: x['complete'] is not None or x['instruct'] is not None)
         hard_leaderboard_df = get_leaderboard_df(
@@ -202,51 +202,23 @@ def get_latest_data_leaderboard(
             cols=COLS,
         )
         HARD_LEADERBOARD_DF = hard_leaderboard_df
-        
-        elo_task_df = datasets.load_dataset(
-            ELO_REPO,
-            "default", 
-            split="task_no_tie", 
-            cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
-            verification_mode="no_checks"
-        ).to_pandas()
-        elo_bench_df = datasets.load_dataset(
-            ELO_REPO,
-            "default", 
-            split="benchmark_tie", 
-            cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
-            verification_mode="no_checks"
-        ).to_pandas()
-        ELO_TASK_DF = elo_task_df
-        ELO_BENCH_DF = elo_bench_df
-        
-        hard_elo_task_df = datasets.load_dataset(
-            HARD_ELO_REPO,
-            "default", 
-            split="task_no_tie", 
-            cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
-            verification_mode="no_checks"
-        ).to_pandas()
-        hard_elo_bench_df = datasets.load_dataset(
-            HARD_ELO_REPO,
-            "default", 
-            split="benchmark_tie", 
-            cache_dir=HF_HOME, 
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset 
-            verification_mode="no_checks"
-        ).to_pandas()
-        HARD_ELO_TASK_DF = hard_elo_task_df
-        HARD_ELO_BENCH_DF = hard_elo_bench_df
+
+        # Comment out Elo dataset loading
+        # elo_task_df = datasets.load_dataset(...)
+        # elo_bench_df = datasets.load_dataset(...)
+        # ELO_TASK_DF = elo_task_df
+        # ELO_BENCH_DF = elo_bench_df
+        # hard_elo_task_df = datasets.load_dataset(...)
+        # hard_elo_bench_df = datasets.load_dataset(...)
+        # HARD_ELO_TASK_DF = hard_elo_task_df
+        # HARD_ELO_BENCH_DF = hard_elo_bench_df
         
         complete_solve_df = datasets.load_dataset(
             SOLVE_REPO,
             "default",
             split="complete",
             cache_dir=HF_HOME,
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS,
             verification_mode="no_checks"
         ).to_pandas()
         instruct_solve_df = datasets.load_dataset(
@@ -254,7 +226,7 @@ def get_latest_data_leaderboard(
             "default",
             split="instruct",
             cache_dir=HF_HOME,
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS,
             verification_mode="no_checks"
         ).to_pandas()
         COMPLETE_SOLVE_DF = complete_solve_df
@@ -265,7 +237,7 @@ def get_latest_data_leaderboard(
             "default",
             split="complete",
             cache_dir=HF_HOME,
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS,
             verification_mode="no_checks"
         ).to_pandas()
         hard_instruct_solve_df = datasets.load_dataset(
@@ -273,7 +245,7 @@ def get_latest_data_leaderboard(
             "default",
             split="instruct",
             cache_dir=HF_HOME,
-            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS, # Uses the cached dataset
+            download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS,
             verification_mode="no_checks"
         ).to_pandas()        
         HARD_COMPLETE_SOLVE_DF = hard_complete_solve_df
@@ -283,18 +255,17 @@ def get_latest_data_leaderboard(
 
     else:
         LEADERBOARD_DF = leaderboard_initial_df
-        # HARD_LEADERBOARD_DF = hard_leaderboard_initial_df
-        ELO_TASK_DF = elo_task_df
-        # ELO_BENCH_DF = elo_bench_df
-        # HARD_ELO_TASK_DF = hard_elo_task_df
-        HARD_ELO_BENCH_DF = hard_elo_bench_df
+        HARD_LEADERBOARD_DF = hard_leaderboard_initial_df
+        # ELO_TASK_DF = elo_task_df  # Comment out
+        # ELO_BENCH_DF = elo_bench_df  # Comment out
+        # HARD_ELO_TASK_DF = hard_elo_task_df  # Comment out
+        # HARD_ELO_BENCH_DF = hard_elo_bench_df  # Comment out
         COMPLETE_SOLVE_DF = complete_solve_df
-        # INSTRUCT_SOLVE_DF = instruct_solve_df
-        # HARD_COMPLETE_SOLVE_DF = hard_complete_solve_df
+        INSTRUCT_SOLVE_DF = instruct_solve_df
+        HARD_COMPLETE_SOLVE_DF = hard_complete_solve_df
         HARD_INSTRUCT_SOLVE_DF = hard_instruct_solve_df
         
-    return (LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
-    # return (HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
+    return (LEADERBOARD_DF, HARD_LEADERBOARD_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
 
 
 def init_space():
@@ -303,19 +274,19 @@ def init_space():
     # Always redownload the leaderboard DataFrame
     global LEADERBOARD_DF
     global HARD_LEADERBOARD_DF
-    global ELO_TASK_DF
-    global ELO_BENCH_DF
-    global HARD_ELO_TASK_DF
-    global HARD_ELO_BENCH_DF
+    # global ELO_TASK_DF  # Comment out
+    # global ELO_BENCH_DF  # Comment out
+    # global HARD_ELO_TASK_DF  # Comment out
+    # global HARD_ELO_BENCH_DF  # Comment out
     global COMPLETE_SOLVE_DF
     global INSTRUCT_SOLVE_DF
     global HARD_COMPLETE_SOLVE_DF
     global HARD_INSTRUCT_SOLVE_DF
     
-    LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = get_latest_data_leaderboard()
+    LEADERBOARD_DF, HARD_LEADERBOARD_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = get_latest_data_leaderboard()
     # HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = get_latest_data_leaderboard()
 
-    return (LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
+    return (LEADERBOARD_DF, HARD_LEADERBOARD_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
     # return (HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF)
 
 # Initialize VoteManager
@@ -331,10 +302,7 @@ def init_space():
 
 # Calls the init_space function with the `full_init` parameter determined by the `do_full_init` variable.
 # This initializes various DataFrames used throughout the application, with the level of initialization detail controlled by the `do_full_init` flag.
-LEADERBOARD_DF, HARD_LEADERBOARD_DF, ELO_TASK_DF, \
-ELO_BENCH_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, \
-COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, \
-HARD_INSTRUCT_SOLVE_DF = init_space()
+LEADERBOARD_DF, HARD_LEADERBOARD_DF, COMPLETE_SOLVE_DF, INSTRUCT_SOLVE_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = init_space()
 # HARD_LEADERBOARD_DF, HARD_ELO_TASK_DF, HARD_ELO_BENCH_DF, HARD_COMPLETE_SOLVE_DF, HARD_INSTRUCT_SOLVE_DF = init_space()
 
 # Data processing for plots now only on demand in the respective Gradio tab
@@ -399,7 +367,6 @@ with main_block as demo:
                     - <u>Instruct</u> (🔥Vibe Check🔥): Code Generation based on the (less verbose) NL-oriented instructions. This split tests if the models are really capable enough to understand human intents to code.
                 - `Complete` and `Instruct` represent the calibrated Pass@1 score on the BigCodeBench benchmark splits.
                 - `Average` is the average of `Complete` and `Instruct` when both are available.
-                - `Elo Rating` represents the task-level Bootstrap of Maximum Likelihood Elo rating on the Complete + Instruct splits. The rating starts from 1000 and is bootstrapped 500 times. We only consider the models having both `Complete` and `Instruct` scores.
                 - `#Act Params (B)` is the number of activated model parameters during inference.
                 - Model providers have the responsibility to avoid data contamination. Models trained on close data can be affected by contamination.
                 - For more details check the 📝 About section.
@@ -407,20 +374,21 @@ with main_block as demo:
                     elem_classes="markdown-text",
                 )
             
-            with gr.TabItem("📊 Elo Rating", id="hard_elo"):
-                with gr.Column():
-                    with gr.Group():
-                        gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_")
-                        hard_task_elo_map = gr.Plot()
-                        hard_elo_task_gr = init_others(HARD_ELO_TASK_DF)
-                        demo.load(plot_elo_mle, [hard_elo_task_gr],
-                                    hard_task_elo_map)
-                    with gr.Group():
-                        gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)")
-                        hard_bench_elo_map = gr.Plot()
-                        hard_elo_bench_gr = init_others(HARD_ELO_BENCH_DF)
-                        demo.load(plot_elo_mle, [hard_elo_bench_gr],
-                                    hard_bench_elo_map)
+            # Comment out or remove the Elo Rating tab
+            # with gr.TabItem("📊 Elo Rating", id="hard_elo"):
+            #     with gr.Column():
+            #         with gr.Group():
+            #             gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_")
+            #             hard_task_elo_map = gr.Plot()
+            #             hard_elo_task_gr = init_others(HARD_ELO_TASK_DF)
+            #             demo.load(plot_elo_mle, [hard_elo_task_gr],
+            #                         hard_task_elo_map)
+            #         with gr.Group():
+            #             gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)")
+            #             hard_bench_elo_map = gr.Plot()
+            #             hard_elo_bench_gr = init_others(HARD_ELO_BENCH_DF)
+            #             demo.load(plot_elo_mle, [hard_elo_bench_gr],
+            #                         hard_bench_elo_map)
                         
             with gr.TabItem("🧩 Solve Rate", id="hard_solve"):
                 with gr.Column():
@@ -448,27 +416,26 @@ with main_block as demo:
                     - <u>Complete</u>: Code Completion based on the (verbose) structured docstring. This variant tests if the models are good at coding.
                     - <u>Instruct</u> (🔥Vibe Check🔥): Code Generation based on the (less verbose) NL-oriented instructions. This variant tests if the models are really capable enough to understand human intents to code.
                 - `complete` and `instruct` represent the calibrated Pass@1 score on the BigCodeBench benchmark variants.
-                - `elo_mle` represents the task-level Bootstrap of Maximum Likelihood Elo rating on the BigCodeBench-Complete split. The rating starts from 1000 and is bootstrapped 500 times.
-                - `size` is the amount of activated model weight during inference.
                 - Model providers have the responsibility to avoid data contamination. Models trained on close data can be affected by contamination.
                 - For more details check the 📝 About section.
                 """,
                     elem_classes="markdown-text",
                 )
             
-            with gr.TabItem("📊 Elo Rating", id="full_elo"):
-                with gr.Column():
-                    with gr.Group():
-                        
-                        gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_")
-                        task_elo_map = gr.Plot()
-                        elo_task_gr = init_others(ELO_TASK_DF)
-                        demo.load(plot_elo_mle, [elo_task_gr], task_elo_map)
-                    with gr.Group():
-                        gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)")
-                        bench_elo_map = gr.Plot()
-                        elo_bench_gr = init_others(ELO_BENCH_DF)
-                        demo.load(plot_elo_mle, [elo_bench_gr], bench_elo_map)
+            # Comment out or remove the Elo Rating tab
+            # with gr.TabItem("📊 Elo Rating", id="full_elo"):
+            #     with gr.Column():
+            #         with gr.Group():
+            #             
+            #             gr.Markdown("## (Task-level, No Tie, BigCodeBench-Complete) -- _Recommended_")
+            #             task_elo_map = gr.Plot()
+            #             elo_task_gr = init_others(ELO_TASK_DF)
+            #             demo.load(plot_elo_mle, [elo_task_gr], task_elo_map)
+            #         with gr.Group():
+            #             gr.Markdown("## (Benchmark-level, BigCodeBench-Complete)")
+            #             bench_elo_map = gr.Plot()
+            #             elo_bench_gr = init_others(ELO_BENCH_DF)
+            #             demo.load(plot_elo_mle, [elo_bench_gr], bench_elo_map)
                     
             with gr.TabItem("🧩 Solve Rate", id="full_solve"):
                 with gr.Column():
@@ -602,7 +569,7 @@ with main_block as demo:
                 show_copy_button=True,
             )
                     
-    main_block.load(fn=get_latest_data_leaderboard, inputs=[leaderboard, hard_leaderboard, elo_task_gr, elo_bench_gr, hard_elo_task_gr, hard_elo_bench_gr, complete_solve_gr, instruct_solve_gr, hard_complete_solve_gr, hard_instruct_solve_gr], outputs=[leaderboard, hard_leaderboard, elo_task_gr, elo_bench_gr, hard_elo_task_gr, hard_elo_bench_gr, complete_solve_gr, instruct_solve_gr, hard_complete_solve_gr, hard_instruct_solve_gr])
+    main_block.load(fn=get_latest_data_leaderboard, inputs=[leaderboard, hard_leaderboard, complete_solve_gr, instruct_solve_gr, hard_complete_solve_gr, hard_instruct_solve_gr], outputs=[leaderboard, hard_leaderboard, complete_solve_gr, instruct_solve_gr, hard_complete_solve_gr, hard_instruct_solve_gr])
     # main_block.load(fn=get_latest_data_leaderboard, inputs=[hard_leaderboard, hard_elo_task_gr, hard_elo_bench_gr, hard_complete_solve_gr, hard_instruct_solve_gr], outputs=[hard_leaderboard, hard_elo_task_gr, hard_elo_bench_gr, hard_complete_solve_gr, hard_instruct_solve_gr])
     # leaderboard.change(fn=get_latest_data_queue, inputs=None, outputs=[finished_eval_table, running_eval_table, pending_eval_table])
     # pending_eval_table.change(fn=vote_manager.create_request_vote_df, inputs=[pending_eval_table], outputs=[pending_eval_table_votes])

src/display/utils.py

@@ -54,7 +54,7 @@ column_map = {
     "complete": "Complete",
     "instruct": "Instruct",
     "average": "Average",
-    "elo_mle": "Elo Rating",
+    # "elo_mle": "Elo Rating",
     "link": "Link",
     "act_param": "#Act Params (B)",
     "size": "#Params (B)",
@@ -96,7 +96,7 @@ auto_eval_column_dict.append(["size_range", ColumnContent, ColumnContent(column_
 auto_eval_column_dict.append(["complete", ColumnContent, ColumnContent(column_map["complete"], "number", True)])
 auto_eval_column_dict.append(["instruct", ColumnContent, ColumnContent(column_map["instruct"], "number", True)])
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent(column_map["average"], "number", True)])
-auto_eval_column_dict.append(["elo_mle", ColumnContent, ColumnContent(column_map["elo_mle"], "number", True)])
+# auto_eval_column_dict.append(["elo_mle", ColumnContent, ColumnContent(column_map["elo_mle"], "number", True)])
 
 # Model information
 auto_eval_column_dict.append(["act_param", ColumnContent, ColumnContent(column_map["act_param"], "number", True)])

src/tools/plots.py

@@ -3,18 +3,18 @@ import plotly.express as px
 import numpy as np
 
 
-def plot_elo_mle(df):
-    fig = px.scatter(df, x="model", y="rating", error_y="error_y",
-                     error_y_minus="error_y_minus",
-                    #  title="Bootstrap of Elo MLE Estimates (BigCodeBench-Complete)"
-                     )
-    fig.update_layout(xaxis_title="Model", 
-                      yaxis_title="Rating",
-                      autosize=True,
-                    #   width=1300,
-                    #   height=900,
-                      )
-    return fig
+# def plot_elo_mle(df):
+#     fig = px.scatter(df, x="model", y="rating", error_y="error_y",
+#                      error_y_minus="error_y_minus",
+#                     #  title="Bootstrap of Elo MLE Estimates (BigCodeBench-Complete)"
+#                      )
+#     fig.update_layout(xaxis_title="Model", 
+#                       yaxis_title="Rating",
+#                       autosize=True,
+#                     #   width=1300,
+#                     #   height=900,
+#                       )
+#     return fig
 
 
 def plot_solve_rate(df, task, rows=30, cols=38):