Revert "Add Bradley-Terry rankings tab (#26)"

This reverts commit e08b34a37838e80b978b0620ee54c7e0d7f86463.

app.py

@@ -42,7 +42,6 @@ def patch_gradio_leaderboard():
 
 patch_gradio_leaderboard()
 
-import pandas as pd
 import gradio as gr
 from apscheduler.schedulers.background import BackgroundScheduler
 from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
@@ -51,25 +50,58 @@ from huggingface_hub import HfApi
 from src.charts import (
     clean_markdown_link,
     create_leaderboard_benchmark_plot,
-    create_score_vs_cost_plot,
+    create_performance_vs_resource_plot,
 )
 from src.display.text_blocks import (
     HOW_TO_USE_TEXT,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
 )
-from src.leaderboard import get_benchmark_names, get_benchmark_run_df, get_score_vs_cost_df
-from src.rankings import load_and_rank, RANK_BY_OPTIONS
+from src.leaderboard import (
+    EFFICIENCY_RESOURCE_METRICS,
+    get_analysis_df,
+    get_benchmark_names,
+    get_benchmark_run_df,
+    get_efficiency_df,
+    get_token_efficiency_table_df,
+)
 
 REPO_ID = "taagarwa/coding-agent-leaderboard"
 TOKEN = os.environ.get("HF_TOKEN")
 API = HfApi(token=TOKEN)
 COLOR_BY_CHOICES = ["Model", "Harness"]
-COLOR_PALETTE_CHOICES = ["Citrus", "Okabe-Ito", "High contrast", "Rainbow"]
+EFFICIENCY_COLOR_BY_CHOICES = ["Model", "Harness"]
+COLOR_PALETTE_CHOICES = [
+    "Citrus",
+    "Okabe-Ito",
+    "High contrast",
+    "Rainbow",
+    "Grayscale",
+    "Viridis",
+    "Plasma",
+    "Cividis",
+]
 DEFAULT_COLOR_PALETTE = "Citrus"
 PLOT_BACKGROUND_CHOICES = ["Dark", "White"]
 DEFAULT_PLOT_BACKGROUND = "Dark"
-FORCE_DARK_MODE_HEAD = """
+RESPONSIVE_PLOT_MIN_HEIGHT_PX = 420
+RESPONSIVE_PLOT_CSS = f"""
+<style>
+.responsive-plot {{
+  overflow-x: auto;
+  width: 100%;
+  min-height: {RESPONSIVE_PLOT_MIN_HEIGHT_PX}px;
+}}
+.responsive-plot .plot-container,
+.responsive-plot .js-plotly-plot,
+.responsive-plot .plotly-graph-div {{
+  width: 100% !important;
+  min-height: {RESPONSIVE_PLOT_MIN_HEIGHT_PX}px;
+}}
+</style>
+"""
+FORCE_DARK_MODE_HEAD = (
+    """
 <script>
 (() => {
   const url = new URL(window.location.href);
@@ -80,6 +112,8 @@ FORCE_DARK_MODE_HEAD = """
 })();
 </script>
 """
+    + RESPONSIVE_PLOT_CSS
+)
 
 
 def restart_space():
@@ -89,7 +123,7 @@ def restart_space():
 BENCHMARK_NAMES = get_benchmark_names()
 DEFAULT_BENCHMARK = BENCHMARK_NAMES[0] if BENCHMARK_NAMES else None
 BENCHMARK_RUN_DF = get_benchmark_run_df()
-SCORE_VS_COST_DF = get_score_vs_cost_df()
+ANALYSIS_DF = get_analysis_df()
 
 
 def render_leaderboard_benchmark_plot(
@@ -107,19 +141,39 @@ def render_leaderboard_benchmark_plot(
     )
 
 
-def render_score_vs_cost_plot(
+
+def render_efficiency(
     benchmark_name,
+    token_metric,
     color_by,
+    x_scale,
+    show_pareto_frontier,
+    show_labels,
     color_palette=DEFAULT_COLOR_PALETTE,
     plot_background=DEFAULT_PLOT_BACKGROUND,
 ):
-    return create_score_vs_cost_plot(
-        SCORE_VS_COST_DF,
+    plot_df = get_efficiency_df(
         benchmark_name=benchmark_name,
+        resource_metric=token_metric,
+        analysis_df=ANALYSIS_DF,
+    )
+    table_df = get_token_efficiency_table_df(
+        benchmark_name=benchmark_name,
+        analysis_df=ANALYSIS_DF,
+    )
+    exclusion_count = plot_df.attrs.get("exclusion_count", 0)
+    note = f"{exclusion_count} runs excluded for this benchmark because {token_metric.lower()} was missing or non-positive."
+    figure = create_performance_vs_resource_plot(
+        plot_df,
+        resource_metric=token_metric,
         color_by=color_by,
+        x_scale=x_scale,
+        show_pareto_frontier=show_pareto_frontier,
+        show_labels=show_labels,
         palette_name=color_palette,
         background_name=plot_background,
     )
+    return figure, table_df, note
 
 
 def build_header_html(df):
@@ -244,128 +298,95 @@ with demo:
                         outputs=plot,
                     )
 
-        with gr.Tab("💰 Cost vs Performance"):
-            cost_benchmark = gr.Dropdown(
-                choices=BENCHMARK_NAMES,
-                value=DEFAULT_BENCHMARK,
-                label="Benchmark",
+        with gr.Tab("⚡ Efficiency") as efficiency_tab:
+            gr.Markdown(
+                "### Efficiency\n"
+                "Compare score with resource use within one benchmark. The x-axis can show total tokens, "
+                "cost per task, or agent execution time per task. The dashed Pareto frontier connects runs "
+                "for which no other displayed run uses an equal or lower amount of the selected resource "
+                "while achieving an equal or higher score. Tokens Per Solved Task remains in the ranking "
+                "table as a reference metric."
             )
             with gr.Row():
-                cost_color_by = gr.Radio(
-                    choices=COLOR_BY_CHOICES,
+                efficiency_benchmark = gr.Dropdown(
+                    choices=BENCHMARK_NAMES,
+                    value=BENCHMARK_NAMES[0] if BENCHMARK_NAMES else None,
+                    label="Benchmark",
+                )
+                efficiency_metric = gr.Dropdown(
+                    choices=list(EFFICIENCY_RESOURCE_METRICS),
+                    value="Total tokens",
+                    label="Resource metric",
+                )
+                efficiency_color_by = gr.Radio(
+                    choices=EFFICIENCY_COLOR_BY_CHOICES,
                     value="Model",
                     label="Color by",
-                    elem_classes="color-control",
                 )
-                cost_palette = gr.Dropdown(
+                efficiency_scale = gr.Radio(choices=["Log", "Linear"], value="Log", label="X-axis scale")
+            with gr.Row():
+                efficiency_pareto = gr.Checkbox(value=True, label="Show Pareto frontier")
+                efficiency_labels = gr.Checkbox(value=False, label="Show point labels")
+                efficiency_palette = gr.Dropdown(
                     choices=COLOR_PALETTE_CHOICES,
                     value=DEFAULT_COLOR_PALETTE,
                     label="Color palette",
-                    elem_classes="color-control",
                 )
-                cost_background = gr.Dropdown(
+                efficiency_background = gr.Dropdown(
                     choices=PLOT_BACKGROUND_CHOICES,
                     value=DEFAULT_PLOT_BACKGROUND,
                     label="Image background",
-                    elem_classes="color-control",
                 )
-            score_vs_cost_plot = gr.Plot(
-                value=render_score_vs_cost_plot(
-                    DEFAULT_BENCHMARK,
-                    "Model",
-                    DEFAULT_COLOR_PALETTE,
-                    DEFAULT_PLOT_BACKGROUND,
-                ),
+
+            initial_efficiency = render_efficiency(
+                BENCHMARK_NAMES[0] if BENCHMARK_NAMES else None,
+                "Total tokens",
+                "Model",
+                "Log",
+                True,
+                False,
+                DEFAULT_COLOR_PALETTE,
+                DEFAULT_PLOT_BACKGROUND,
+            )
+            efficiency_note = gr.Markdown(initial_efficiency[2])
+            efficiency_plot = gr.Plot(
+                value=initial_efficiency[0],
                 show_label=False,
                 elem_classes="responsive-plot",
             )
-            cost_benchmark.change(
-                fn=render_score_vs_cost_plot,
-                inputs=[cost_benchmark, cost_color_by, cost_palette, cost_background],
-                outputs=score_vs_cost_plot,
-            )
-            cost_color_by.change(
-                fn=render_score_vs_cost_plot,
-                inputs=[cost_benchmark, cost_color_by, cost_palette, cost_background],
-                outputs=score_vs_cost_plot,
-            )
-            cost_palette.change(
-                fn=render_score_vs_cost_plot,
-                inputs=[cost_benchmark, cost_color_by, cost_palette, cost_background],
-                outputs=score_vs_cost_plot,
-            )
-            cost_background.change(
-                fn=render_score_vs_cost_plot,
-                inputs=[cost_benchmark, cost_color_by, cost_palette, cost_background],
-                outputs=score_vs_cost_plot,
+            gr.Markdown("#### Efficiency ranking")
+            efficiency_table = gr.Dataframe(
+                value=initial_efficiency[1],
+                interactive=False,
+                show_label=False,
             )
 
-        with gr.Tab("📊 Rankings"):
-            gr.Markdown("### Bradley-Terry Paired Comparison Rankings")
-            gr.Markdown("Rankings computed across all benchmarks using [paired comparison methods](https://github.com/erikerlandson/paired-comparison-ranking). Handles missing data and inconsistent orderings.")
-
-            rank_csv = pd.read_csv("results.csv")
-            rank_csv = rank_csv.dropna(subset=["metrics.score"])
-            rank_model_choices = sorted(rank_csv["model.name"].unique().tolist())
-            rank_harness_choices = sorted(rank_csv["harness.name"].unique().tolist())
-            rank_benchmark_choices = sorted(rank_csv["benchmark.name"].unique().tolist())
-
-            with gr.Row():
-                rank_by = gr.Dropdown(
-                    choices=list(RANK_BY_OPTIONS.keys()),
-                    value="Benchmark Score",
-                    label="Rank by",
-                )
-                rank_oss_models = gr.Checkbox(value=False, label="Open models only")
-                rank_oss_harnesses = gr.Checkbox(value=False, label="Open harnesses only")
-            with gr.Row():
-                rank_benchmark_filter = gr.CheckboxGroup(
-                    choices=rank_benchmark_choices,
-                    value=rank_benchmark_choices,
-                    label="Benchmarks",
-                )
-            with gr.Row():
-                rank_model_filter = gr.CheckboxGroup(
-                    choices=rank_model_choices,
-                    value=rank_model_choices,
-                    label="Models",
-                )
-            with gr.Row():
-                rank_harness_filter = gr.CheckboxGroup(
-                    choices=rank_harness_choices,
-                    value=rank_harness_choices,
-                    label="Harnesses",
-                )
-
-            harness_df_init, model_df_init, pair_df_init = load_and_rank("results.csv")
-
-            gr.Markdown("#### Harness Rankings")
-            harness_table = gr.Dataframe(value=harness_df_init, interactive=False)
-            gr.Markdown("#### Model Rankings")
-            model_table = gr.Dataframe(value=model_df_init, interactive=False)
-            gr.Markdown("#### (Model, Harness) Rankings")
-            pair_table = gr.Dataframe(value=pair_df_init, interactive=False)
-
-            def update_rankings(rank_by_val, oss_models, oss_harnesses, benchmarks, models, harnesses):
-                h, m, p = load_and_rank(
-                    "results.csv",
-                    open_models_only=oss_models,
-                    open_harnesses_only=oss_harnesses,
-                    benchmarks=benchmarks,
-                    models=models,
-                    harnesses=harnesses,
-                    rank_by=rank_by_val,
-                )
-                return h, m, p
-
-            ranking_inputs = [rank_by, rank_oss_models, rank_oss_harnesses, rank_benchmark_filter, rank_model_filter, rank_harness_filter]
-            for control in ranking_inputs:
+            efficiency_controls = [
+                efficiency_benchmark,
+                efficiency_metric,
+                efficiency_color_by,
+                efficiency_scale,
+                efficiency_pareto,
+                efficiency_labels,
+                efficiency_palette,
+                efficiency_background,
+            ]
+            for control in efficiency_controls:
                 control.change(
-                    fn=update_rankings,
-                    inputs=ranking_inputs,
-                    outputs=[harness_table, model_table, pair_table],
+                    fn=render_efficiency,
+                    inputs=efficiency_controls,
+                    outputs=[efficiency_plot, efficiency_table, efficiency_note],
                 )
 
+            # Gradio initially lays out hidden tabs without reliable dimensions. Re-render
+            # once Efficiency becomes visible so Plotly can autosize against the real
+            # container instead of keeping the hidden-tab geometry.
+            efficiency_tab.select(
+                fn=render_efficiency,
+                inputs=efficiency_controls,
+                outputs=[efficiency_plot, efficiency_table, efficiency_note],
+            )
+
         with gr.Tab("🏃 Benchmark Runs"):
             benchmark_runs = init_benchmark_runs(BENCHMARK_RUN_DF)
 

requirements.txt

@@ -15,5 +15,3 @@ tqdm
 transformers
 tokenizers>=0.15.0
 sentencepiece
-choix
-scipy

src/rankings.py

@@ -1,152 +0,0 @@
-from typing import Any
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-import choix
-
-
-def prepare_ranking_data(
-    df: pd.DataFrame,
-    catcol: str | list[str],
-    metcol: str,
-    descending: bool = False,
-    eqvcol: str | list[str] = [],
-) -> tuple[list[tuple[int, int]], list[Any], dict]:
-    ndata = df.shape[0]
-    if ndata < 2:
-        raise ValueError("Not enough data to prepare ranking comparisons")
-    catcol = catcol if isinstance(catcol, list) else [catcol]
-    eqvcol = eqvcol if isinstance(eqvcol, list) else [eqvcol]
-    ncat = len(catcol)
-    neqv = len(eqvcol)
-    tcols = catcol + eqvcol + [metcol]
-    t = list(df[tcols].itertuples(index=False, name=None))
-    metvals = [x[-1] for x in t]
-    if ncat > 1:
-        catvals = [x[:ncat] for x in t]
-    else:
-        catvals = [x[0] for x in t]
-    if neqv > 1:
-        eqvvals = [x[ncat : ncat + neqv] for x in t]
-    elif neqv == 1:
-        eqvvals = [x[ncat] for x in t]
-    else:
-        eqvvals = ["[ALL]"] * ndata
-    umap = dict([(y, x) for x, y in enumerate(sorted(set(catvals)))])
-    cats = sorted(umap.keys())
-    eqvcats = sorted(set(eqvvals))
-    eqvdata = {}
-    for eqv in eqvcats:
-        eqvdata[eqv] = [[] for _ in range(len(cats))]
-    compvals = [(umap[c], m, e) for c, m, e in zip(catvals, metvals, eqvvals)]
-    for category, metric, equivalence in compvals:
-        eqvdata[equivalence][category].append(metric)
-    comps = []
-    for i in range(ndata):
-        ic, im, ie = compvals[i]
-        for j in range(i):
-            jc, jm, je = compvals[j]
-            if ie != je:
-                continue
-            if im == jm:
-                continue
-            iwin = im < jm if descending else im > jm
-            if iwin:
-                comps.append((ic, jc))
-            else:
-                comps.append((jc, ic))
-    return comps, cats, eqvdata
-
-
-def ranking_dataframe(cats, params, eqvdata) -> pd.DataFrame:
-    ranking = np.argsort(params)[::-1]
-    rows = []
-    for rank, idx in enumerate(ranking, start=1):
-        row = {
-            "Rank": rank,
-            "Category": cats[idx] if not isinstance(cats[idx], tuple) else " + ".join(cats[idx]),
-        }
-        for k in sorted(eqvdata.keys()):
-            mets = eqvdata[k][idx]
-            row[k] = round(float(np.mean(mets)), 3) if len(mets) > 0 else None
-        rows.append(row)
-    return pd.DataFrame(rows)
-
-
-RANK_BY_OPTIONS = {
-    "Benchmark Score": ("metrics.score", False),
-    "Mean Cost Per Task (USD)": ("metrics.mean_cost_usd_per_task", True),
-    "Mean Tokens Per Task": ("metrics.mean_tokens_per_task", True),
-}
-
-
-def compute_ranking(df, catcol, metcol="metrics.score", descending=False, eqvcol="benchmark.name"):
-    comps, cats, eqvdata = prepare_ranking_data(
-        df, catcol, metcol, descending=descending, eqvcol=eqvcol
-    )
-    params = choix.ilsr_pairwise(len(cats), comps, alpha=1e-3)
-    return ranking_dataframe(cats, params, eqvdata)
-
-
-def rank_harnesses(df: pd.DataFrame, metcol="metrics.score", descending=False) -> pd.DataFrame:
-    return compute_ranking(df, "harness.name", metcol, descending)
-
-
-def rank_models(df: pd.DataFrame, metcol="metrics.score", descending=False) -> pd.DataFrame:
-    return compute_ranking(df, "model.name", metcol, descending)
-
-
-def rank_pairs(df: pd.DataFrame, metcol="metrics.score", descending=False) -> pd.DataFrame:
-    return compute_ranking(df, ["model.name", "harness.name"], metcol, descending)
-
-
-def _load_csv(csv_path: str | Path = "results.csv") -> pd.DataFrame:
-    df = pd.read_csv(csv_path)
-    df = df.dropna(subset=["metrics.score"])
-    df = df.loc[df["metrics.score"] > 0]
-    return df.reset_index(drop=True)
-
-
-def _empty_table():
-    return pd.DataFrame({"Rank": [], "Category": []})
-
-
-def load_and_rank(
-    csv_path: str | Path = "results.csv",
-    open_models_only: bool = False,
-    open_harnesses_only: bool = False,
-    benchmarks: list[str] | None = None,
-    models: list[str] | None = None,
-    harnesses: list[str] | None = None,
-    rank_by: str = "Benchmark Score",
-) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
-    df = _load_csv(csv_path)
-    if open_models_only:
-        df = df.loc[df["model.is_oss"] == True]
-    if open_harnesses_only:
-        df = df.loc[df["harness.is_oss"] == True]
-    if benchmarks is not None:
-        df = df.loc[df["benchmark.name"].isin(benchmarks)]
-    if models is not None:
-        df = df.loc[df["model.name"].isin(models)]
-    if harnesses is not None:
-        df = df.loc[df["harness.name"].isin(harnesses)]
-
-    metcol, descending = RANK_BY_OPTIONS.get(rank_by, ("metrics.score", False))
-    df = df.dropna(subset=[metcol])
-    if descending:
-        df = df.loc[df[metcol] > 0]
-    df = df.reset_index(drop=True)
-
-    if len(df) < 2:
-        empty = _empty_table()
-        return empty, empty, empty
-
-    results = []
-    for rank_fn in (rank_harnesses, rank_models, rank_pairs):
-        try:
-            results.append(rank_fn(df, metcol, descending))
-        except ValueError:
-            results.append(_empty_table())
-    return results[0], results[1], results[2]