scoring.py

"""Grading + leaderboard logic. Pure pandas/sklearn (no Gradio/network) so it's testable.
Ranked metric is Brier (lower is better); AUROC/Accuracy are shown for information."""
import numpy as np, pandas as pd
from sklearn.metrics import roc_auc_score

REQUIRED_COLS = {"id", "p_correct"}
COLUMNS = ["team", "submitted_at", "Brier", "AUROC", "Accuracy", "n"]

def grade(submission: pd.DataFrame, labels: pd.Series) -> dict:
    """submission: columns id,p_correct.  labels: Series indexed by id with 0/1.
    Returns {Brier, AUROC, Accuracy, n}. Raises ValueError on a malformed submission."""
    missing_cols = REQUIRED_COLS - set(submission.columns)
    if missing_cols:
        raise ValueError(f"submission needs columns id,p_correct (missing {sorted(missing_cols)})")
    sub = submission.copy()
    sub["id"] = sub["id"].astype(str)
    sub = sub.drop_duplicates("id").set_index("id")

    ids = labels.index.astype(str)
    missing = [i for i in ids if i not in sub.index]
    if missing:
        raise ValueError(f"submission is missing {len(missing)} of {len(ids)} ids "
                         f"(e.g. {missing[:3]}). Score every record in the test set.")

    p = pd.to_numeric(sub.loc[ids, "p_correct"], errors="coerce").to_numpy(dtype=float)
    if np.isnan(p).any():
        raise ValueError("p_correct has non-numeric / missing values")
    p = np.clip(p, 0.0, 1.0)
    y = labels.loc[ids].to_numpy(dtype=int)

    brier = float(np.mean((p - y) ** 2))
    acc   = float(np.mean((p >= 0.5).astype(int) == y))
    auroc = float(roc_auc_score(y, p)) if len(np.unique(y)) == 2 else float("nan")
    return {"Brier": round(brier, 4), "AUROC": round(auroc, 4),
            "Accuracy": round(acc, 4), "n": int(len(y))}

def append_submission(board: pd.DataFrame, team: str, metrics: dict, ts: str) -> pd.DataFrame:
    """Keep each team's BEST (lowest Brier) row."""
    row = {"team": team, "submitted_at": ts, **metrics}
    board = pd.concat([board, pd.DataFrame([row])], ignore_index=True)
    board = board.sort_values("Brier", ascending=True).drop_duplicates("team", keep="first")
    return board.reset_index(drop=True)

def render_leaderboard(board: pd.DataFrame) -> pd.DataFrame:
    cols = ["rank", "team", "Brier", "AUROC", "Accuracy", "submitted_at"]
    if board.empty:
        return pd.DataFrame(columns=cols)
    b = board.sort_values("Brier", ascending=True).reset_index(drop=True)
    b.insert(0, "rank", np.arange(1, len(b) + 1))
    return b[cols]