consolidate

app.py

@@ -1,7 +1,8 @@
 import gradio as gr
 from gradio_leaderboard import Leaderboard
 import pandas as pd
-from huggingface_hub import snapshot_download
+from huggingface_hub import snapshot_download, create_repo, RepoNotFoundError
+import os
 
 from src.about import (
     INTRODUCTION_TEXT,
@@ -14,13 +15,13 @@ from src.display.utils import (
     COLS,
     AutoEvalColumn,
 )
-from src.envs import API, EVAL_RESULTS_PATH, RESULTS_REPO, TOKEN
+from src.envs import API, EVAL_RESULTS_PATH, RESULTS_REPO, TOKEN, OWNER
 from src.populate import get_leaderboard_df
 from src.evaluation.dynamic_eval import run_dynamic_perplexity_eval
 
 def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
+    if dataframe is None:
+        raise ValueError("Leaderboard DataFrame is None.")
     
     return Leaderboard(
         dataframe,
@@ -42,14 +43,33 @@ def run_perplexity_test(model_name, revision, precision):
     else:
         return f"❌ Evaluation failed: {result}"
 
-# Initialize results directory
+# Initialize results repository and directory
 try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
+    # Try to download existing repository
+    try:
+        snapshot_download(
+            repo_id=RESULTS_REPO,
+            local_dir=EVAL_RESULTS_PATH,
+            repo_type="dataset",
+            tqdm_class=None,
+            etag_timeout=30,
+            token=TOKEN
+        )
+    except RepoNotFoundError:
+        # Create the repository if it doesn't exist
+        print(f"Creating new results repository: {RESULTS_REPO}")
+        create_repo(
+            repo_id=RESULTS_REPO,
+            repo_type="dataset",
+            private=False,
+            token=TOKEN
+        )
+        # Create local directory
+        os.makedirs(EVAL_RESULTS_PATH, exist_ok=True)
 except Exception as e:
     print(f"Error initializing results: {e}")
+    # Ensure local directory exists even if repo operations fail
+    os.makedirs(EVAL_RESULTS_PATH, exist_ok=True)
 
 # Get initial leaderboard data
 LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)

src/populate.py

@@ -5,13 +5,31 @@ from src.leaderboard.read_evals import get_raw_eval_results
 
 def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path)
-    all_data_json = [v.to_dict() for v in raw_data]
+    try:
+        raw_data = get_raw_eval_results(results_path)
+        all_data_json = [v.to_dict() for v in raw_data]
 
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
+        if not all_data_json:
+            # Create empty DataFrame with correct columns
+            empty_df = pd.DataFrame(columns=cols)
+            # Ensure correct column types
+            empty_df[AutoEvalColumn.average.name] = pd.Series(dtype=float)
+            for col in benchmark_cols:
+                empty_df[col] = pd.Series(dtype=float)
+            return empty_df
 
-    # filter out if perplexity hasn't been evaluated
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
+        df = pd.DataFrame.from_records(all_data_json)
+        df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+        df = df[cols].round(decimals=2)
+
+        # filter out if perplexity hasn't been evaluated
+        df = df[has_no_nan_values(df, benchmark_cols)]
+        return df
+    except Exception as e:
+        print(f"Error creating leaderboard: {e}")
+        # Return empty DataFrame with correct structure
+        empty_df = pd.DataFrame(columns=cols)
+        empty_df[AutoEvalColumn.average.name] = pd.Series(dtype=float)
+        for col in benchmark_cols:
+            empty_df[col] = pd.Series(dtype=float)
+        return empty_df