File size: 2,509 Bytes
7f2e6e6 7774431 7f2e6e6 7774431 7f2e6e6 7774431 7f2e6e6 f7afa88 7774431 f7afa88 7774431 f7afa88 7774431 f7afa88 7f2e6e6 7774431 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | import os
import pandas as pd
try:
from src.display.utils import fields
except ImportError:
try:
from src.utils import fields
except ImportError:
try:
from display.utils import fields
except ImportError:
from utils import fields
try:
from src.leaderboard.read_evals import get_raw_eval_results
except ImportError:
try:
from src.read_evals import get_raw_eval_results
except ImportError:
try:
from leaderboard.read_evals import get_raw_eval_results
except ImportError:
from read_evals import get_raw_eval_results
def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
raw_data = get_raw_eval_results(results_path, requests_path)
all_col_contents = fields()
all_col_names = [c.name for c in all_col_contents]
if not raw_data:
return pd.DataFrame(columns=all_col_names)
all_data_json = [v.to_dict() for v in raw_data]
df = pd.DataFrame.from_records(all_data_json)
# Ensure all expected columns exist and are strictly typed
for col_content in all_col_contents:
if col_content.name not in df.columns:
df[col_content.name] = 0.0 if col_content.type == "number" else ""
elif col_content.type == "number":
df[col_content.name] = pd.to_numeric(df[col_content.name], errors="coerce").fillna(0.0)
else:
df[col_content.name] = df[col_content.name].fillna("").astype(str)
df = df[all_col_names]
# Sort ascending by risk/composite score (lower score = lower risk = rank higher)
for col in all_col_names:
if "composite" in col.lower() or "risk" in col.lower() or "score" in col.lower():
df = df.sort_values(by=[col], ascending=True)
break
return df.reset_index(drop=True)
def get_top_3_eval_cards(results_path: str):
raw_data = get_raw_eval_results(results_path, "")
if not raw_data:
return []
sorted_res = sorted(raw_data, key=lambda x: x.composite_score)
top_3 = []
for r in sorted_res[:3]:
top_3.append({
"model_name": r.full_model,
"org": r.org,
"factuality": r.results.get("factuality", 0.0),
"blind_fraction": r.results.get("blind_fraction", 0.0),
"composite_score": r.composite_score,
"run_id": f"REV-{r.revision}",
"date": r.date,
})
return top_3 |