File size: 2,354 Bytes
22df562 7939a4f 22df562 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | """
Data population functions for MUSEval Leaderboard
"""
import pandas as pd
from typing import Dict, List, Any, Optional
from .load_results import load_results_with_metadata, create_overall_table
def get_leaderboard_df(results_path: str, requests_path: str, eval_cols: List[str], benchmark_cols: List[str]) -> pd.DataFrame:
"""Get leaderboard dataframe"""
# Use our existing load_results function
results = load_results_with_metadata()
if not results:
return pd.DataFrame()
return create_overall_table()
def get_model_info_df(results_path: str, requests_path: str) -> pd.DataFrame:
"""Get model information dataframe"""
results = load_results_with_metadata()
if not results:
return pd.DataFrame()
# Extract unique model information
model_info = {}
for result in results:
model = result["model"]
if model not in model_info:
model_info[model] = {
"model": model,
"organization": result["submitter"],
"submission_date": result["submission_date"],
"task": result.get("task", ""),
"dataset_version": result.get("dataset_version", ""),
"paper_url": result.get("paper_url", ""),
"code_url": result.get("code_url", ""),
"model_type": "Foundation Model", # Default
"testdata_leakage": "No" # Default
}
return pd.DataFrame(list(model_info.values()))
def get_merged_df(leaderboard_df: pd.DataFrame, model_info_df: pd.DataFrame) -> pd.DataFrame:
"""Merge leaderboard and model info dataframes"""
if leaderboard_df.empty or model_info_df.empty:
return leaderboard_df
# Merge on model name
merged = pd.merge(leaderboard_df, model_info_df, on="model", how="left")
# Add rank column
if 'MAE' in merged.columns:
merged['Rank'] = merged['MAE'].rank(method='min').astype(int)
# Move Rank to front
cols = ['Rank'] + [col for col in merged.columns if col != 'Rank']
merged = merged[cols]
return merged
def get_evaluation_queue_df(requests_path: str, eval_cols: List[str]) -> tuple:
"""Get evaluation queue dataframes"""
# Return empty dataframes for now
return pd.DataFrame(), pd.DataFrame(), pd.DataFrame()
|