Upload app.py with huggingface_hub

app.py

@@ -5,25 +5,48 @@ import pandas as pd
 import plotly.graph_objects as go
 
 RESULTS_REPO = "happynood/quantcall-results"
-RESULTS_FILE = "data/results.csv"
+RESULTS_FILE = "data/leaderboard.csv"
 GITHUB_URL = "https://github.com/Happynood/quant-toolcall-bench"
+SUITE_URL = "https://huggingface.co/datasets/happynood/quantcall-suite"
+RESULTS_URL = "https://huggingface.co/datasets/happynood/quantcall-results"
 CONTRIBUTING_URL = f"{GITHUB_URL}/blob/main/CONTRIBUTING.md"
 
-DISPLAY_COLS = ["Model", "Quant", "Backend", "SVR", "TSA", "AC", "Abst", "FCR", "ΔFCR", "VRAM (GB)", "η"]
+DISPLAY_COLS = [
+    "Model",
+    "Quant",
+    "Backend",
+    "Decoding",
+    "Tier",
+    "Seeds",
+    "SVR",
+    "TSA",
+    "AC",
+    "Abst",
+    "FCR",
+    "FCR 95% CI",
+    "VRAM (GB)",
+    "η",
+    "ΔFCR %",
+    "Baseline",
+]
 CSV_TO_DISPLAY = {
     "model": "Model",
     "quant": "Quant",
     "backend": "Backend",
-    "svr": "SVR",
-    "tsa": "TSA",
-    "ac": "AC",
-    "abstention": "Abst",
-    "fcr": "FCR",
-    "delta_fcr": "ΔFCR",
+    "decoding": "Decoding",
+    "tier": "Tier",
+    "n_seeds": "Seeds",
+    "svr_mean": "SVR",
+    "tsa_mean": "TSA",
+    "ac_mean": "AC",
+    "abstention_mean": "Abst",
+    "fcr_mean": "FCR",
     "vram_gb": "VRAM (GB)",
     "eta": "η",
+    "delta_fcr_rel": "ΔFCR %",
+    "baseline_quant": "Baseline",
 }
-FLOAT_COLS = {"SVR", "TSA", "AC", "Abst", "FCR", "ΔFCR", "VRAM (GB)", "η"}
+FLOAT_COLS = {"SVR", "TSA", "AC", "Abst", "FCR", "VRAM (GB)", "η"}
 FILTER_BACKENDS = ["All", "llama-cpp", "transformers", "vllm", "openai"]
 FILTER_QUANTS = ["All", "fp16", "Q8_0", "Q5_K_M", "Q4_K_M", "AWQ", "GPTQ"]
 
@@ -38,7 +61,7 @@ def _empty_df() -> pd.DataFrame:
 
 
 def _load_results_df() -> pd.DataFrame:
-    """Download results CSV via hf_hub_download; return empty DataFrame on any error."""
+    """Download the aggregated leaderboard CSV; return empty DataFrame on any error."""
     try:
         from huggingface_hub import hf_hub_download
 
@@ -50,12 +73,29 @@ def _load_results_df() -> pd.DataFrame:
         df = pd.read_csv(path)
         if df.empty:
             return _empty_df()
+
+        for col in ("fcr_ci_low", "fcr_ci_high"):
+            if col not in df.columns:
+                df[col] = pd.NA
+        df["FCR 95% CI"] = df.apply(
+            lambda r: (
+                f"[{r['fcr_ci_low']:.3f}, {r['fcr_ci_high']:.3f}]"
+                if pd.notna(r["fcr_ci_low"]) and pd.notna(r["fcr_ci_high"])
+                else ""
+            ),
+            axis=1,
+        )
+        if "delta_fcr_rel" in df.columns:
+            df["delta_fcr_rel"] = pd.to_numeric(df["delta_fcr_rel"], errors="coerce") * 100.0
+
         df = df.rename(columns=CSV_TO_DISPLAY)
         present = [c for c in DISPLAY_COLS if c in df.columns]
         df = df[present].sort_values("FCR", ascending=False).reset_index(drop=True)
         for col in FLOAT_COLS:
             if col in df.columns:
                 df[col] = pd.to_numeric(df[col], errors="coerce").round(4)
+        if "ΔFCR %" in df.columns:
+            df["ΔFCR %"] = pd.to_numeric(df["ΔFCR %"], errors="coerce").round(1)
         return df
     except Exception:
         return _empty_df()
@@ -83,6 +123,25 @@ def _pareto_chart(df: pd.DataFrame) -> go.Figure:
         return fig
 
     valid = df.dropna(subset=["VRAM (GB)", "FCR"])
+    if valid.empty:
+        fig.add_annotation(
+            text="No VRAM measurements yet — submit a PR to populate this chart!",
+            x=0.5,
+            y=0.5,
+            xref="paper",
+            yref="paper",
+            showarrow=False,
+            font={"size": 14, "color": "#6b7280"},
+        )
+        fig.update_layout(
+            xaxis={"visible": False},
+            yaxis={"visible": False},
+            plot_bgcolor="white",
+            paper_bgcolor="white",
+            height=420,
+        )
+        return fig
+
     if "Model" in valid.columns and "Quant" in valid.columns:
         label = valid["Model"].astype(str) + " " + valid["Quant"].astype(str)
     else:
@@ -132,11 +191,20 @@ _initial_df = _load_results_df()
 
 with gr.Blocks(title="QuantCall Leaderboard") as demo:
     gr.Markdown(
-        f"# 📊 QuantCall — Function-Calling Reliability Under Quantization\n\n"
-        f"> Does quantizing your model quietly break its ability to call tools?\n\n"
-        f"**[GitHub]({GITHUB_URL})** · "
-        f"**[Submit results]({CONTRIBUTING_URL})** · "
-        f"**[Eval suite](https://huggingface.co/datasets/happynood/quantcall-suite)**"
+        "# 📊 QuantCall — Function-Calling Reliability Under Quantization\n\n"
+        "> Does quantizing your model quietly break its ability to call tools?"
+    )
+    gr.HTML(
+        f"""
+        <div style="display:flex; gap:0.75rem; align-items:center; margin:-0.5rem 0 1rem 0;
+                     font-size:0.95rem;">
+          <a href="{GITHUB_URL}" target="_blank" rel="noopener noreferrer"><b>GitHub</b></a>
+          <span>·</span>
+          <a href="{RESULTS_URL}" target="_blank" rel="noopener noreferrer"><b>Submit results</b></a>
+          <span>·</span>
+          <a href="{SUITE_URL}" target="_blank" rel="noopener noreferrer"><b>Eval suite</b></a>
+        </div>
+        """
     )
 
     df_state = gr.State(_initial_df)
@@ -158,7 +226,7 @@ with gr.Blocks(title="QuantCall Leaderboard") as demo:
         table = gr.DataFrame(
             _initial_df,
             interactive=False,
-            label="Results (sorted by FCR ↓)",
+            label="Results (sorted by FCR ↓, aggregated over seeds — see 'Seeds' column)",
         )
 
         def _update_table(
@@ -206,19 +274,21 @@ with gr.Blocks(title="QuantCall Leaderboard") as demo:
 | **AC** | Argument Correctness — correct argument values (AST-match)? |
 | **Abst** | Abstention Accuracy — correct silence when no tool is relevant? |
 | **FCR** | 0.25 × (SVR + TSA + AC + Abst) |
-| **ΔFCR** | Absolute degradation vs fp16 baseline |
+| **ΔFCR %** | Relative FCR degradation vs the row's `Baseline` quant (see column) |
 | **η** | FCR / peak VRAM (GB) |
+| **Baseline** | The Δ reference quant for that (model, backend, decoding, tier) —
+  fp16 if it fits and was run, otherwise the best available quant that was run |
 
 ## Dataset Tiers
 
 | Tier | Source | License |
 |------|--------|---------|
 | T0 | In-repo smoke (10 instances) | MIT |
-| T1/T2 | BFCL simple/parallel | Apache 2.0 |
+| T1/T2 | BFCL v4 simple/multiple/parallel | Apache 2.0 |
 | T3 | ToolACE | CC-BY-NC 4.0 |
 | T4 | xLAM ungated mirror | NC/gated |
 | T5 | Hermes function-calling v1 | Apache 2.0 (bundles glaive-function-calling-5k; credit both sources) |
-| T6 | BFCL irrelevance (abstention) | Apache 2.0 |
+| T6 | BFCL v4 irrelevance (abstention) | Apache 2.0 |
 
 ## How to Submit