consolidate

app.py

@@ -25,6 +25,10 @@ def init_leaderboard(dataframe):
     if dataframe is None:
         raise ValueError("Leaderboard DataFrame is None.")
     
+    print("\n=== Initializing Leaderboard ===", flush=True)
+    print(f"DataFrame shape: {dataframe.shape}", flush=True)
+    print(f"DataFrame columns: {dataframe.columns.tolist()}", flush=True)
+    
     return Leaderboard(
         value=dataframe,
         select_columns=[c.name for c in fields(AutoEvalColumn) if not c.hidden],
@@ -38,8 +42,10 @@ def init_leaderboard(dataframe):
 
 def refresh_leaderboard():
     """Refresh leaderboard data from disk"""
+    print("\n=== Refreshing Leaderboard ===", flush=True)
     try:
         # Download latest results
+        print("Downloading latest results...", flush=True)
         snapshot_download(
             repo_id=RESULTS_REPO,
             local_dir=EVAL_RESULTS_PATH,
@@ -48,23 +54,34 @@ def refresh_leaderboard():
             etag_timeout=30,
             token=TOKEN
         )
+        print("Download complete", flush=True)
     except Exception as e:
-        print(f"Error refreshing results: {e}")
+        print(f"Error refreshing results: {e}", flush=True)
     
     # Get fresh leaderboard data
+    print("Getting fresh leaderboard data...", flush=True)
     df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
+    print(f"Got DataFrame with shape: {df.shape}", flush=True)
     return init_leaderboard(df)
 
 def run_perplexity_test(model_name, revision, precision):
     """Run perplexity evaluation on demand."""
+    print(f"\n=== Running Perplexity Test ===", flush=True)
+    print(f"Model: {model_name}", flush=True)
+    print(f"Revision: {revision}", flush=True)
+    print(f"Precision: {precision}", flush=True)
+    
     if not model_name:
         return "Please enter a model name.", None
     
     success, result = run_dynamic_perplexity_eval(model_name, revision, precision)
+    print(f"Evaluation result - Success: {success}, Result: {result}", flush=True)
     
     if success:
         # Get updated leaderboard
+        print("Refreshing leaderboard...", flush=True)
         new_leaderboard = refresh_leaderboard()
+        print("Leaderboard refresh complete", flush=True)
         return f"✅ Perplexity evaluation completed!\nPerplexity: {result:.4f}", new_leaderboard
     else:
         return f"❌ Evaluation failed: {result}", None

src/leaderboard/read_evals.py

@@ -76,6 +76,9 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
+        print(f"\nProcessing result for model: {self.full_model}", flush=True)
+        print(f"Raw results: {self.results}", flush=True)
+        
         # Calculate average, handling perplexity (lower is better)
         scores = []
         perplexity_score = None
@@ -90,6 +93,7 @@ class EvalResult:
                 scores.append(score)
         
         average = sum(scores) / len(scores) if scores else 0
+        print(f"Calculated average score: {average}", flush=True)
         
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
@@ -111,13 +115,17 @@ class EvalResult:
         # Add perplexity score with the exact column name from Tasks
         if perplexity_score is not None:
             data_dict[Tasks.task0.value.col_name] = perplexity_score
+            print(f"Added perplexity score {perplexity_score} under column {Tasks.task0.value.col_name}", flush=True)
         else:
             data_dict[Tasks.task0.value.col_name] = None
+            print(f"No perplexity score found for column {Tasks.task0.value.col_name}", flush=True)
 
+        print(f"Final data dict keys: {list(data_dict.keys())}", flush=True)
         return data_dict
 
 def get_raw_eval_results(results_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all perplexity results"""
+    print(f"\nSearching for result files in: {results_path}", flush=True)
     model_result_filepaths = []
 
     for root, _, files in os.walk(results_path):
@@ -128,29 +136,39 @@ def get_raw_eval_results(results_path: str) -> list[EvalResult]:
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
+    print(f"Found {len(model_result_filepaths)} result files", flush=True)
+
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         try:
+            print(f"\nProcessing file: {model_result_filepath}", flush=True)
             # Creation of result
             eval_result = EvalResult.init_from_json_file(model_result_filepath)
+            print(f"Created result object for: {eval_result.full_model}", flush=True)
 
             # Store results of same eval together
             eval_name = eval_result.eval_name
             if eval_name in eval_results.keys():
                 eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+                print(f"Updated existing result for {eval_name}", flush=True)
             else:
                 eval_results[eval_name] = eval_result
+                print(f"Added new result for {eval_name}", flush=True)
         except Exception as e:
-            print(f"Error processing result file {model_result_filepath}: {e}")
+            print(f"Error processing result file {model_result_filepath}: {e}", flush=True)
             continue
 
     results = []
+    print(f"\nProcessing {len(eval_results)} evaluation results", flush=True)
     for v in eval_results.values():
         try:
+            print(f"\nConverting result to dict for: {v.full_model}", flush=True)
             v.to_dict() # we test if the dict version is complete
             results.append(v)
-        except KeyError as e:  # not all eval values present
-            print(f"Error converting result to dict: {e}")
+            print("Successfully converted and added result", flush=True)
+        except KeyError as e:
+            print(f"Error converting result to dict: {e}", flush=True)
             continue
 
+    print(f"\nReturning {len(results)} processed results", flush=True)
     return results

src/populate.py

@@ -5,31 +5,56 @@ from src.leaderboard.read_evals import get_raw_eval_results
 
 def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
+    print("\n=== Starting leaderboard creation ===", flush=True)
+    print(f"Looking for results in: {results_path}", flush=True)
+    print(f"Expected columns: {cols}", flush=True)
+    print(f"Benchmark columns: {benchmark_cols}", flush=True)
+
+    raw_data = get_raw_eval_results(results_path)
+    print(f"\nFound {len(raw_data)} raw results", flush=True)
+
+    all_data_json = [v.to_dict() for v in raw_data]
+    print(f"\nConverted to {len(all_data_json)} JSON records", flush=True)
+    if all_data_json:
+        print("Sample record keys:", list(all_data_json[0].keys()), flush=True)
+
+    if not all_data_json:
+        print("\nNo data found, creating empty DataFrame", flush=True)
+        empty_df = pd.DataFrame(columns=cols)
+        # Ensure correct column types
+        empty_df[AutoEvalColumn.average.name] = pd.Series(dtype=float)
+        for col in benchmark_cols:
+            empty_df[col] = pd.Series(dtype=float)
+        return empty_df
+
+    df = pd.DataFrame.from_records(all_data_json)
+    print("\nCreated DataFrame with columns:", df.columns.tolist(), flush=True)
+    print("DataFrame shape:", df.shape, flush=True)
+
     try:
-        raw_data = get_raw_eval_results(results_path)
-        all_data_json = [v.to_dict() for v in raw_data]
-
-        if not all_data_json:
-            # Create empty DataFrame with correct columns
-            empty_df = pd.DataFrame(columns=cols)
-            # Ensure correct column types
-            empty_df[AutoEvalColumn.average.name] = pd.Series(dtype=float)
-            for col in benchmark_cols:
-                empty_df[col] = pd.Series(dtype=float)
-            return empty_df
-
-        df = pd.DataFrame.from_records(all_data_json)
         df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-        df = df[cols].round(decimals=2)
+        print("\nSorted DataFrame by average", flush=True)
+    except KeyError as e:
+        print(f"\nError sorting DataFrame: {e}", flush=True)
+        print("Available columns:", df.columns.tolist(), flush=True)
 
-        # filter out if perplexity hasn't been evaluated
-        df = df[has_no_nan_values(df, benchmark_cols)]
-        return df
-    except Exception as e:
-        print(f"Error creating leaderboard: {e}")
-        # Return empty DataFrame with correct structure
+    try:
+        df = df[cols].round(decimals=2)
+        print("\nSelected and rounded columns", flush=True)
+    except KeyError as e:
+        print(f"\nError selecting columns: {e}", flush=True)
+        print("Requested columns:", cols, flush=True)
+        print("Available columns:", df.columns.tolist(), flush=True)
+        # Create empty DataFrame with correct structure
         empty_df = pd.DataFrame(columns=cols)
         empty_df[AutoEvalColumn.average.name] = pd.Series(dtype=float)
         for col in benchmark_cols:
             empty_df[col] = pd.Series(dtype=float)
         return empty_df
+
+    # filter out if perplexity hasn't been evaluated
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    print("\nFinal DataFrame shape after filtering:", df.shape, flush=True)
+    print("Final columns:", df.columns.tolist(), flush=True)
+    
+    return df