Deploy read-only benchmark explorer

README.md

@@ -1,15 +1,14 @@
 ---
-title: BasedBench
-emoji: 🏃
-colorFrom: pink
-colorTo: purple
+title: basedBench
 sdk: gradio
-sdk_version: 6.20.0
-python_version: '3.12'
+sdk_version: 6.17.3
 app_file: app.py
 pinned: false
-license: other
-short_description: An evaluation benchmark for meme understanding
+license: mit
+python_version: 3.12
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# basedBench
+
+Read-only explorer and leaderboard for the
+[basedBench dataset](https://huggingface.co/datasets/montagovian/basedBench).

app.py

@@ -0,0 +1,374 @@
+"""Two-tab read-only BasedBench explorer for Hugging Face Spaces."""
+
+from __future__ import annotations
+
+import html
+import secrets
+from typing import Any
+
+import gradio as gr
+
+try:
+    from data import BenchmarkData, load_from_hub
+except ImportError:
+    from space.data import BenchmarkData, load_from_hub
+
+
+DATA: BenchmarkData = load_from_hub()
+
+
+def _escaped(value: Any) -> str:
+    return html.escape(str(value or ""))
+
+
+def _quoted(value: Any) -> str:
+    lines = _escaped(value).splitlines() or [""]
+    return "\n".join(f"> {line}" for line in lines)
+
+
+def _prediction_markdown(post_id: str, selected_model: str) -> str:
+    blocks: list[str] = []
+    for prediction in DATA.predictions(post_id, selected_model):
+        prediction_id = int(prediction["prediction_id"])
+        judgments = DATA.judgments(prediction_id)
+        correct = sum(row.get("verdict") == "correct" for row in judgments)
+        incorrect = sum(row.get("verdict") == "incorrect" for row in judgments)
+        consensus = str(prediction.get("consensus_verdict") or "no consensus")
+        judge_lines = []
+        for judgment in judgments:
+            line = (
+                f"**{_escaped(judgment['judge_model'])}:** "
+                f"{_escaped(judgment['verdict'])}"
+            )
+            if judgment.get("reasoning"):
+                line += "\n\n" + _quoted(judgment["reasoning"])
+            judge_lines.append(line)
+        historical = DATA.historical_judgment_counts.get(prediction_id, 0)
+        history_note = (
+            f"\n\n_{historical} superseded judgment record"
+            f"{'s' if historical != 1 else ''} retained in the dataset._"
+            if historical
+            else ""
+        )
+        judge_details = "\n\n".join(judge_lines) or "_No judge records._"
+        blocks.append(
+            f"### `{_escaped(prediction['model_id'])}`\n\n"
+            f"**Consensus: {consensus}** · {correct} correct / {incorrect} incorrect\n\n"
+            f"<details><summary>Model prediction</summary>\n\n"
+            f"{_escaped(prediction['prediction'])}\n\n</details>\n\n"
+            f"<details><summary>Judge details</summary>\n\n"
+            f"{judge_details}"
+            f"{history_note}\n\n</details>"
+        )
+    return "\n\n---\n\n".join(blocks) or "_No prediction matches this filter._"
+
+
+def _empty_render(position: str = "0 / 0") -> tuple[Any, ...]:
+    return (
+        0,
+        position,
+        gr.update(value=None, visible=False),
+        gr.update(value="_No memes match these filters._", visible=True),
+        gr.update(value="", visible=False),
+        gr.update(value="", visible=False),
+    )
+
+
+def _render(
+    ids: list[str], idx: int, hide_ground_truth: bool, selected_model: str
+) -> tuple[Any, ...]:
+    if not ids:
+        return _empty_render()
+    bounded = max(0, min(int(idx), len(ids) - 1))
+    post_id = ids[bounded]
+    meme = DATA.meme(post_id)
+    info = (
+        f"## {_escaped(meme['title'])}\n\n"
+        f"`r/{_escaped(meme['subreddit'])}` · `{_escaped(post_id)}`"
+    )
+    return (
+        bounded,
+        f"{bounded + 1} / {len(ids)}",
+        gr.update(value=DATA.image(post_id), visible=True),
+        gr.update(value=info, visible=True),
+        gr.update(
+            value=("Ground truth hidden." if hide_ground_truth else meme["ground_truth"]),
+            visible=True,
+        ),
+        gr.update(
+            value=_prediction_markdown(post_id, selected_model),
+            visible=True,
+        ),
+    )
+
+
+def apply_filters(
+    search: str, model_id: str, result: str, hide_ground_truth: bool
+) -> tuple[Any, ...]:
+    ids = DATA.filtered_ids(search, model_id, result)
+    return (ids, *_render(ids, 0, hide_ground_truth, model_id))
+
+
+def step_item(
+    ids: list[str], idx: int, delta: int, hide_ground_truth: bool, model_id: str
+) -> tuple[Any, ...]:
+    return _render(ids, int(idx) + delta, hide_ground_truth, model_id)
+
+
+def random_item(
+    ids: list[str], hide_ground_truth: bool, model_id: str
+) -> tuple[Any, ...]:
+    if not ids:
+        return _empty_render()
+    return _render(ids, secrets.randbelow(len(ids)), hide_ground_truth, model_id)
+
+
+def rerender_item(
+    ids: list[str], idx: int, hide_ground_truth: bool, model_id: str
+) -> tuple[Any, ...]:
+    return _render(ids, idx, hide_ground_truth, model_id)
+
+
+CSS = """
+.gradio-container {
+    max-width: 1180px !important;
+}
+.app-header {
+    align-items: baseline !important;
+    margin-bottom: 4px !important;
+}
+.app-title h1 {
+    margin: 0 !important;
+    line-height: 1.1 !important;
+}
+.app-subtitle {
+    color: var(--body-text-color-subdued) !important;
+    font-size: 14px !important;
+}
+.inspect-toolbar {
+    gap: 8px !important;
+    align-items: center !important;
+    flex-wrap: wrap !important;
+    margin-bottom: 8px !important;
+}
+.inspect-toolbar .block {
+    min-width: 0 !important;
+}
+.nav-button {
+    min-width: 82px !important;
+    max-width: 96px !important;
+}
+.random-button {
+    min-width: 78px !important;
+    max-width: 88px !important;
+}
+.inspect-position {
+    min-width: 72px !important;
+    max-width: 84px !important;
+    text-align: center !important;
+    color: var(--body-text-color-subdued) !important;
+}
+.inspect-position p {
+    margin: 0 !important;
+}
+.meme-image img {
+    width: 100% !important;
+    max-height: 72vh !important;
+    object-fit: contain !important;
+    object-position: top center !important;
+}
+.prediction-panel details {
+    border-top: 1px solid var(--border-color-primary);
+    padding: 8px 0;
+}
+.prediction-panel summary {
+    cursor: pointer;
+    font-weight: 600;
+}
+.leaderboard-table {
+    min-height: 250px !important;
+}
+@media (max-width: 700px) {
+    .gradio-container {
+        padding-left: 10px !important;
+        padding-right: 10px !important;
+    }
+    .inspect-toolbar {
+        gap: 6px !important;
+    }
+    .filter-toolbar .form {
+        display: grid !important;
+        grid-template-columns: minmax(0, 1fr) minmax(0, 1fr) !important;
+        gap: 6px !important;
+        width: 100% !important;
+    }
+    .filter-toolbar .form > .block {
+        flex: none !important;
+        min-width: 0 !important;
+        max-width: none !important;
+        width: 100% !important;
+    }
+    .filter-toolbar .form > .block:first-child,
+    .filter-toolbar .form > .block:last-child {
+        grid-column: 1 / -1 !important;
+    }
+    .nav-button,
+    .random-button {
+        min-width: 70px !important;
+        max-width: none !important;
+        flex: 1 1 auto !important;
+    }
+    .meme-image img {
+        max-height: none !important;
+    }
+}
+"""
+
+
+def build_app() -> gr.Blocks:
+    model_choices = [("All models", "all")] + [(model, model) for model in DATA.models]
+    with gr.Blocks(title="basedBench") as demo:
+        with gr.Row(elem_classes="app-header"):
+            gr.HTML(
+                "<div class='app-title'><h1>basedBench</h1>"
+                "<div class='app-subtitle'>Read-only benchmark explorer</div></div>"
+            )
+
+        with gr.Tabs(selected="inspect"):
+            with gr.Tab("Inspect", id="inspect"):
+                ids_state = gr.State([])
+                idx_state = gr.State(0)
+
+                with gr.Row(elem_classes=["inspect-toolbar", "filter-toolbar"]):
+                    search = gr.Textbox(
+                        placeholder="Search title, source, ID, or ground truth",
+                        label="Search",
+                        show_label=False,
+                        min_width=260,
+                        scale=3,
+                    )
+                    model = gr.Dropdown(
+                        choices=model_choices,
+                        value="all",
+                        label="Model",
+                        show_label=False,
+                        min_width=210,
+                        scale=2,
+                    )
+                    result = gr.Dropdown(
+                        choices=[
+                            ("Any result", "all"),
+                            ("Consensus correct", "correct"),
+                            ("Consensus incorrect", "incorrect"),
+                            ("Judge disagreement", "disagreement"),
+                        ],
+                        value="all",
+                        label="Result",
+                        show_label=False,
+                        min_width=180,
+                        scale=2,
+                    )
+                    hide_ground_truth = gr.Checkbox(
+                        label="Hide ground truth",
+                        value=False,
+                        min_width=150,
+                        scale=1,
+                    )
+
+                with gr.Row(elem_classes="inspect-toolbar"):
+                    previous = gr.Button("Previous", elem_classes="nav-button")
+                    random_button = gr.Button("Random", elem_classes="random-button")
+                    position = gr.Markdown("0 / 0", elem_classes="inspect-position")
+                    next_button = gr.Button("Next", elem_classes="nav-button")
+
+                with gr.Row(equal_height=False):
+                    with gr.Column(scale=1, min_width=320):
+                        image = gr.Image(
+                            label="Meme",
+                            type="pil",
+                            interactive=False,
+                            elem_classes="meme-image",
+                        )
+                    with gr.Column(scale=1, min_width=320):
+                        info = gr.Markdown()
+                        ground_truth = gr.Textbox(
+                            label="Ground Truth",
+                            lines=5,
+                            interactive=False,
+                        )
+                        predictions = gr.Markdown(elem_classes="prediction-panel")
+
+                render_outputs = [
+                    idx_state,
+                    position,
+                    image,
+                    info,
+                    ground_truth,
+                    predictions,
+                ]
+                filter_outputs = [ids_state, *render_outputs]
+                filter_inputs = [search, model, result, hide_ground_truth]
+
+                demo.load(apply_filters, inputs=filter_inputs, outputs=filter_outputs)
+                search.submit(apply_filters, inputs=filter_inputs, outputs=filter_outputs)
+                model.change(apply_filters, inputs=filter_inputs, outputs=filter_outputs)
+                result.change(apply_filters, inputs=filter_inputs, outputs=filter_outputs)
+                previous.click(
+                    lambda ids, idx, hidden, selected: step_item(
+                        ids, idx, -1, hidden, selected
+                    ),
+                    inputs=[ids_state, idx_state, hide_ground_truth, model],
+                    outputs=render_outputs,
+                )
+                next_button.click(
+                    lambda ids, idx, hidden, selected: step_item(
+                        ids, idx, 1, hidden, selected
+                    ),
+                    inputs=[ids_state, idx_state, hide_ground_truth, model],
+                    outputs=render_outputs,
+                )
+                random_button.click(
+                    random_item,
+                    inputs=[ids_state, hide_ground_truth, model],
+                    outputs=render_outputs,
+                )
+                hide_ground_truth.change(
+                    rerender_item,
+                    inputs=[ids_state, idx_state, hide_ground_truth, model],
+                    outputs=render_outputs,
+                )
+
+            with gr.Tab("Leaderboard"):
+                gr.Markdown(
+                    f"**Snapshot:** `{DATA.snapshot_id}` · "
+                    f"**Memes:** {len(DATA.post_ids):,} · "
+                    f"**Predictions:** {len(DATA.predictions_by_id):,}"
+                )
+                gr.Dataframe(
+                    value=DATA.leaderboard_rows(),
+                    headers=[
+                        "Model",
+                        "Correct",
+                        "Incorrect",
+                        "Total",
+                        "Accuracy",
+                        "Judge agreement",
+                    ],
+                    datatype=["str", "number", "number", "number", "str", "str"],
+                    interactive=False,
+                    wrap=True,
+                    elem_classes="leaderboard-table",
+                )
+                gr.Markdown(
+                    "Consensus requires at least two matching judge votes. "
+                    "Judge agreement is the stricter rate where all latest votes match."
+                )
+
+    return demo
+
+
+demo = build_app()
+
+
+if __name__ == "__main__":
+    demo.launch(css=CSS)

data.py

@@ -0,0 +1,187 @@
+"""Normalized dataset loading and indexing for the read-only Space."""
+
+from __future__ import annotations
+
+import os
+from collections import defaultdict
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+
+DEFAULT_DATASET_REPO = "montagovian/basedBench"
+
+
+def _column(table: Any, name: str) -> list[Any]:
+    try:
+        return list(table[name])
+    except (KeyError, TypeError):
+        return [row[name] for row in table]
+
+
+class BenchmarkData:
+    """In-memory indexes over the four normalized dataset configs."""
+
+    def __init__(
+        self,
+        memes: Any,
+        predictions: Iterable[Mapping[str, Any]],
+        judgments: Iterable[Mapping[str, Any]],
+        leaderboard: Iterable[Mapping[str, Any]],
+    ) -> None:
+        self._memes = memes
+        post_ids = [str(value) for value in _column(memes, "post_id")]
+        titles = [str(value) for value in _column(memes, "title")]
+        subreddits = [str(value) for value in _column(memes, "subreddit")]
+        ground_truths = [str(value) for value in _column(memes, "ground_truth")]
+        snapshot_ids = [str(value) for value in _column(memes, "snapshot_id")]
+
+        self.post_ids = post_ids
+        self._row_index = {post_id: idx for idx, post_id in enumerate(post_ids)}
+        self._meta = {
+            post_id: {
+                "post_id": post_id,
+                "title": titles[idx],
+                "subreddit": subreddits[idx],
+                "ground_truth": ground_truths[idx],
+                "snapshot_id": snapshot_ids[idx],
+            }
+            for idx, post_id in enumerate(post_ids)
+        }
+
+        self.predictions_by_post: dict[str, list[dict[str, Any]]] = defaultdict(list)
+        self.predictions_by_id: dict[int, dict[str, Any]] = {}
+        for source in predictions:
+            row = dict(source)
+            prediction_id = int(row["prediction_id"])
+            post_id = str(row["post_id"])
+            self.predictions_by_id[prediction_id] = row
+            self.predictions_by_post[post_id].append(row)
+        for rows in self.predictions_by_post.values():
+            rows.sort(key=lambda row: str(row["model_id"]))
+
+        self.latest_judgments: dict[int, list[dict[str, Any]]] = defaultdict(list)
+        self.historical_judgment_counts: dict[int, int] = defaultdict(int)
+        for source in judgments:
+            row = dict(source)
+            prediction_id = int(row["prediction_id"])
+            if bool(row.get("is_latest")):
+                self.latest_judgments[prediction_id].append(row)
+            else:
+                self.historical_judgment_counts[prediction_id] += 1
+        for rows in self.latest_judgments.values():
+            rows.sort(key=lambda row: str(row["judge_model"]))
+
+        self.leaderboard = [dict(row) for row in leaderboard]
+        self.leaderboard.sort(
+            key=lambda row: (-float(row["accuracy"]), str(row["model_id"]))
+        )
+        self.models = sorted(
+            {
+                str(row["model_id"])
+                for rows in self.predictions_by_post.values()
+                for row in rows
+            }
+        )
+
+    @property
+    def snapshot_id(self) -> str:
+        if not self.post_ids:
+            return ""
+        return str(self._meta[self.post_ids[0]]["snapshot_id"])
+
+    def meme(self, post_id: str) -> dict[str, Any]:
+        return self._meta[post_id]
+
+    def image(self, post_id: str) -> Any:
+        return self._memes[self._row_index[post_id]]["image"]
+
+    def predictions(self, post_id: str, model_id: str = "all") -> list[dict[str, Any]]:
+        rows = self.predictions_by_post.get(post_id, [])
+        if model_id == "all":
+            return rows
+        return [row for row in rows if str(row["model_id"]) == model_id]
+
+    def judgments(self, prediction_id: int) -> list[dict[str, Any]]:
+        return self.latest_judgments.get(prediction_id, [])
+
+    def filtered_ids(
+        self,
+        search: str = "",
+        model_id: str = "all",
+        result: str = "all",
+    ) -> list[str]:
+        needle = search.strip().casefold()
+        matches: list[str] = []
+        for post_id in self.post_ids:
+            meta = self._meta[post_id]
+            if needle and needle not in " ".join(
+                (
+                    post_id,
+                    str(meta["title"]),
+                    str(meta["subreddit"]),
+                    str(meta["ground_truth"]),
+                )
+            ).casefold():
+                continue
+
+            predictions = self.predictions(post_id, model_id)
+            if model_id != "all" and not predictions:
+                continue
+            if result == "correct" and not any(
+                row.get("consensus_verdict") == "correct" for row in predictions
+            ):
+                continue
+            if result == "incorrect" and not any(
+                row.get("consensus_verdict") == "incorrect" for row in predictions
+            ):
+                continue
+            if result == "disagreement" and not any(
+                len(
+                    {
+                        judgment.get("verdict")
+                        for judgment in self.judgments(int(row["prediction_id"]))
+                    }
+                )
+                > 1
+                for row in predictions
+            ):
+                continue
+            matches.append(post_id)
+        return matches
+
+    def leaderboard_rows(self) -> list[list[Any]]:
+        return [
+            [
+                row["model_id"],
+                int(row["correct"]),
+                int(row["incorrect"]),
+                int(row["total"]),
+                f"{float(row['accuracy']) * 100:.1f}%",
+                (
+                    f"{int(row['unanimous_agreements'])}/"
+                    f"{int(row['judged_by_multiple'])} "
+                    f"({float(row['agreement_rate']) * 100:.1f}%)"
+                ),
+            ]
+            for row in self.leaderboard
+        ]
+
+
+def load_from_hub(repo_id: str | None = None) -> BenchmarkData:
+    """Load the published snapshot directly from the Hub, not dataset-server."""
+    from datasets import load_dataset
+
+    repo = repo_id or os.getenv("HF_DATASET_REPO", DEFAULT_DATASET_REPO)
+    token = os.getenv("HF_TOKEN") or os.getenv("HF_API_KEY")
+    kwargs = {"token": token} if token else {}
+    try:
+        memes = load_dataset(repo, "memes", split="train", **kwargs)
+        predictions = load_dataset(repo, "predictions", split="train", **kwargs)
+        judgments = load_dataset(repo, "judgments", split="train", **kwargs)
+        leaderboard = load_dataset(repo, "leaderboard", split="train", **kwargs)
+    except Exception as exc:
+        raise RuntimeError(
+            f"Unable to load {repo}. For a private dataset, add an HF_TOKEN "
+            "with read access to the Space secrets."
+        ) from exc
+    return BenchmarkData(memes, predictions, judgments, leaderboard)

requirements.txt

@@ -0,0 +1,3 @@
+datasets==4.8.4
+huggingface_hub==1.8.0
+Pillow==12.2.0