version v0.1

debug_df.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import pandas as pd
+from src.display.utils import AutoEvalColumn
+from src.leaderboard.read_evals import get_raw_eval_results
+
+# 设置路径
+EVAL_RESULTS_PATH = "./eval-results"
+EVAL_REQUESTS_PATH = "./eval-queue"
+
+# 获取列定义
+cols = [c.name for c in AutoEvalColumn.__dict__.values() if hasattr(c, "name") and not getattr(c, "hidden", False)]
+benchmark_cols = ["NQ", "TriviaQA", "PopQA", "HotpotQA", "2wiki", "Musique", "Bamboogle", "FictionalHot"]
+
+print("Expected columns from AutoEvalColumn:")
+for c in AutoEvalColumn.__dict__.values():
+    if hasattr(c, "name"):
+        print(f"  {c.name}")
+
+print("\nTrying to get raw data...")
+try:
+    raw_data = get_raw_eval_results(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH)
+    print(f"Found {len(raw_data)} raw data entries")
+
+    if raw_data:
+        print("\nFirst entry structure:")
+        first_entry = raw_data[0]
+        print(f"  eval_name: {first_entry.eval_name}")
+        print(f"  full_model: {first_entry.full_model}")
+        print(f"  results: {first_entry.results}")
+
+        # 转换为字典
+        all_data_json = [v.to_dict() for v in raw_data]
+        df = pd.DataFrame.from_records(all_data_json)
+
+        print(f"\nDataFrame shape: {df.shape}")
+        print(f"DataFrame columns: {list(df.columns)}")
+
+        if len(df) > 0:
+            print("\nFirst row:")
+            print(df.iloc[0])
+
+            print(f"\nLooking for '{AutoEvalColumn.average.name}' column...")
+            if AutoEvalColumn.average.name in df.columns:
+                print(f"Found '{AutoEvalColumn.average.name}' column")
+                print(f"Average values: {df[AutoEvalColumn.average.name].tolist()}")
+            else:
+                print(f"ERROR: '{AutoEvalColumn.average.name}' column not found!")
+                print("Available columns:", list(df.columns))
+    else:
+        print("No raw data found!")
+
+except Exception as e:
+    print(f"Error occurred: {e}")
+    import traceback
+
+    traceback.print_exc()

src/populate.py

@@ -14,35 +14,15 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
 
-    # If empty (e.g., first run on HF Spaces), return empty frame with expected columns
-    if df.empty:
-        return pd.DataFrame(columns=cols)
+    # Add ranking based on average score
+    df[AutoEvalColumn.rank.name] = range(1, len(df) + 1)
 
-    # Ensure Average exists; compute from available benchmark columns if missing
-    average_col = AutoEvalColumn.average.name
-    present_bench_cols = [c for c in benchmark_cols if c in df.columns]
-    if average_col not in df.columns and len(present_bench_cols) > 0:
-        df[average_col] = df[present_bench_cols].mean(axis=1)
-
-    # Sort if Average present
-    if average_col in df.columns:
-        df = df.sort_values(by=[average_col], ascending=False)
-
-    # Add rank if Average present
-    if average_col in df.columns:
-        df[AutoEvalColumn.rank.name] = range(1, len(df) + 1)
-
-    # Add any missing columns as NA and reorder
-    for c in cols:
-        if c not in df.columns:
-            df[c] = pd.NA
     df = df[cols].round(decimals=2)
 
-    # Filter NaNs only on existing benchmark columns
-    safe_benchmark_cols = [c for c in benchmark_cols if c in df.columns]
-    if len(safe_benchmark_cols) > 0:
-        df = df[has_no_nan_values(df, safe_benchmark_cols)]
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
     return df