Updated init_space() mostly

app.py

@@ -1,4 +1,5 @@
 import os
+import logging
 import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
@@ -47,6 +48,7 @@ from src.submission.submit import add_new_eval
 from src.tools.collections import update_collections
 from src.tools.plots import create_metric_plot_obj, create_plot_df, create_scores_df
 
+
 # Start ephemeral Spaces on PRs (see config in README.md)
 enable_space_ci()
 
@@ -55,64 +57,48 @@ def restart_space():
     API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
 
 
-def init_space(full_init: bool = True):
-    if full_init:
-        try:
-            print(EVAL_REQUESTS_PATH)
-            snapshot_download(
-                repo_id=QUEUE_REPO,
-                local_dir=EVAL_REQUESTS_PATH,
-                repo_type="dataset",
-                tqdm_class=None,
-                etag_timeout=30,
-                max_workers=8,
-            )
-        except Exception:
-            restart_space()
-        try:
-            print(DYNAMIC_INFO_PATH)
-            snapshot_download(
-                repo_id=DYNAMIC_INFO_REPO,
-                local_dir=DYNAMIC_INFO_PATH,
-                repo_type="dataset",
-                tqdm_class=None,
-                etag_timeout=30,
-                max_workers=8,
-            )
-        except Exception:
-            restart_space()
+def download_dataset(repo_id, local_dir, repo_type="dataset", max_attempts=3):
+    """Attempt to download dataset with retries."""
+    attempt = 0
+    while attempt < max_attempts:
         try:
-            print(EVAL_RESULTS_PATH)
+            print(f"Downloading {repo_id} to {local_dir}")
             snapshot_download(
-                repo_id=RESULTS_REPO,
-                local_dir=EVAL_RESULTS_PATH,
-                repo_type="dataset",
+                repo_id=repo_id,
+                local_dir=local_dir,
+                repo_type=repo_type,
                 tqdm_class=None,
                 etag_timeout=30,
                 max_workers=8,
             )
-        except Exception:
-            restart_space()
+            return
+        except Exception as e:
+            logging.error(f"Error downloading {repo_id}: {e}")
+            attempt += 1
+            if attempt == max_attempts:
+                restart_space()
+                break
 
-    raw_data, original_df = get_leaderboard_df(
+
+def init_space(full_init: bool = True):
+    """Initializes the application space, loading only necessary data."""
+    if full_init:
+        download_dataset(QUEUE_REPO, EVAL_REQUESTS_PATH)
+        download_dataset(DYNAMIC_INFO_REPO, DYNAMIC_INFO_PATH)
+        download_dataset(RESULTS_REPO, EVAL_RESULTS_PATH)
+
+    raw_data, leaderboard_df = get_leaderboard_df(
         results_path=EVAL_RESULTS_PATH,
         requests_path=EVAL_REQUESTS_PATH,
         dynamic_path=DYNAMIC_INFO_FILE_PATH,
         cols=COLS,
         benchmark_cols=BENCHMARK_COLS,
     )
-    update_collections(original_df.copy())
-    leaderboard_df = original_df.copy()
+    update_collections(leaderboard_df)
 
-    plot_df = create_plot_df(create_scores_df(raw_data))
+    eval_queue_dfs = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
-    (
-        finished_eval_queue_df,
-        running_eval_queue_df,
-        pending_eval_queue_df,
-    ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-    return leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
+    return leaderboard_df, raw_data, eval_queue_dfs
 
 
 # Convert the environment variable "LEADERBOARD_FULL_INIT" to a boolean value, defaulting to True if the variable is not set.
@@ -121,9 +107,14 @@ do_full_init = os.getenv("LEADERBOARD_FULL_INIT", "True") == "True"
 
 # Calls the init_space function with the `full_init` parameter determined by the `do_full_init` variable.
 # This initializes various DataFrames used throughout the application, with the level of initialization detail controlled by the `do_full_init` flag.
-leaderboard_df, original_df, plot_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = (
-    init_space(full_init=do_full_init)
-)
+leaderboard_df, raw_data, eval_queue_dfs = init_space(full_init=do_full_init)
+finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = eval_queue_dfs
+
+
+# Data processing for plots now only on demand in the respective Gradio tab
+def load_and_create_plots():
+    plot_df = create_plot_df(create_scores_df(raw_data))
+    return plot_df
 
 
 # Searching and filtering
@@ -344,7 +335,8 @@ with demo:
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[COLS],
+                # value=original_df[COLS],
+                value=leaderboard_df[COLS],  # UPDATED
                 headers=COLS,
                 datatype=TYPES,
                 visible=False,
@@ -406,6 +398,8 @@ with demo:
         with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=2):
             with gr.Row():
                 with gr.Column():
+                    # UPDATED
+                    plot_df = load_and_create_plots()
                     chart = create_metric_plot_obj(
                         plot_df,
                         [AutoEvalColumn.average.name],
@@ -413,12 +407,15 @@ with demo:
                     )
                     gr.Plot(value=chart, min_width=500)
                 with gr.Column():
+                    # UPDATED
+                    plot_df = load_and_create_plots()
                     chart = create_metric_plot_obj(
                         plot_df,
                         BENCHMARK_COLS,
                         title="Top Scores and Human Baseline Over Time (from last update)",
                     )
                     gr.Plot(value=chart, min_width=500)
+
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 

src/populate.py

@@ -15,12 +15,12 @@ def get_leaderboard_df(
     raw_data = get_raw_eval_results(results_path=results_path, requests_path=requests_path, dynamic_path=dynamic_path)
     all_data_json = [v.to_dict() for v in raw_data]
     all_data_json.append(baseline_row)
-    print([data for data in all_data_json if data["model_name_for_query"] == "databricks/dbrx-base"])
+    # print([data for data in all_data_json if data["model_name_for_query"] == "databricks/dbrx-base"])
     filter_models_flags(all_data_json)
 
     df = pd.DataFrame.from_records(all_data_json)
-    print(df.columns)
-    print(df[df["model_name_for_query"] == "databricks/dbrx-base"])
+    # print(df.columns)
+    # print(df[df["model_name_for_query"] == "databricks/dbrx-base"])
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)