Rank submissions by F1 globally; drop Category and Scaffold columns

app.py

@@ -141,10 +141,10 @@ def _access_label(access: str | None) -> str:
 
 
 def leaderboard_dataframe(submissions: list[dict[str, Any]]) -> pd.DataFrame:
-    """Build a single unified leaderboard DataFrame from every submission file."""
+    """Build a single unified leaderboard DataFrame, globally ranked by F1 desc."""
     if not submissions:
         return pd.DataFrame(columns=[
-            "Category", "Agent", "Model", "Access", "Scaffold",
+            "Rank", "Agent", "Model", "Access",
             "F1", "Precision", "Recall", "EM", "LLM Judge",
             "Org", "Date",
         ])
@@ -156,11 +156,9 @@ def leaderboard_dataframe(submissions: list[dict[str, Any]]) -> pd.DataFrame:
         efficiency = s.get("efficiency", {})
 
         rows.append({
-            "Category":  meta.get("category", "Submission"),
             "Agent":     meta.get("agent_name", "—"),
             "Model":     meta.get("base_model", "—"),
             "Access":    _access_label(meta.get("access")),
-            "Scaffold":  meta.get("scaffold", "—"),
             "F1":        overall.get("f1"),
             "Precision": overall.get("precision"),
             "Recall":    overall.get("recall"),
@@ -180,15 +178,16 @@ def leaderboard_dataframe(submissions: list[dict[str, Any]]) -> pd.DataFrame:
         if col in df.columns and df[col].isna().all():
             df = df.drop(columns=[col])
 
-    # Category order: LLM Baseline first, Agent Framework second, everything
-    # else after. Within each category, sort by LLM Judge desc then F1 desc.
-    cat_order = {"LLM Baseline": 0, "Agent Framework": 1}
-    df["__cat_order"] = df["Category"].map(cat_order).fillna(2)
+    # Global ranking by F1 desc, with LLM Judge as tiebreaker.
     df = df.sort_values(
-        by=["__cat_order", "LLM Judge", "F1"],
-        ascending=[True, False, False],
+        by=["F1", "LLM Judge"],
+        ascending=[False, False],
         na_position="last",
-    ).drop(columns=["__cat_order"]).reset_index(drop=True)
+    ).reset_index(drop=True)
+
+    # Assign rank medals for the top 3, numeric rank for everyone else.
+    medals = {0: "🥇 1", 1: "🥈 2", 2: "🥉 3"}
+    df.insert(0, "Rank", [medals.get(i, str(i + 1)) for i in range(len(df))])
 
     return df
 
@@ -295,11 +294,11 @@ def build_app() -> gr.Blocks:
             # -------------------------------------------------------------
             with gr.Tab("🏆 Leaderboard"):
                 gr.Markdown(
-                    "Results on the **DeepSynth test set** (80 tasks, Pass@1). "
-                    "F1 / Precision / Recall measure prediction quality against gold answers; "
-                    "**LLM Judge** reports average precision under semantic matching. "
-                    "Submissions are grouped into LLM Baselines (no tools, no scaffold) and "
-                    "Agent Frameworks (tool-using agents).",
+                    "Results on the **DeepSynth test set** (80 tasks, Pass@1), "
+                    "ranked by **F1** score (LLM Judge used as tiebreaker). "
+                    "F1 / Precision / Recall measure prediction quality against gold "
+                    "answers; **LLM Judge** reports average precision under semantic "
+                    "matching. 🔒 = closed model, 🔓 = open-weights.",
                     elem_classes=["section-header"],
                 )
                 gr.Dataframe(
@@ -445,4 +444,4 @@ def build_app() -> gr.Blocks:
 
 
 if __name__ == "__main__":
-    build_app().launch()
+    build_app().launch()