Deploy from 1ad58abad9750ed5d30bb5f033c99a815a3813a2

README.md

@@ -1,48 +1,93 @@
 ---
-title: Mlx Benchmarks Viewer
-emoji: 🥇
-colorFrom: green
+title: MLX Benchmarks Viewer
+emoji: 📊
+colorFrom: blue
 colorTo: indigo
 sdk: gradio
+sdk_version: "6.13.0"
 app_file: app.py
+python_version: "3.11"
 pinned: true
 license: apache-2.0
-short_description: Duplicate this leaderboard to initialize your own!
-sdk_version: 5.43.1
+short_description: Interactive viewer for the MLX Benchmarks dataset
 tags:
-- leaderboard
+  - benchmarks
+  - mlx
+  - apple-silicon
+  - lm-eval
+  - visualization
+models:
+  - mlx-community/Qwen3.5-9B-MLX-4bit
+datasets:
+  - JacobPEvans/mlx-benchmarks
 ---
 
-# Start the configuration
-
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
-
-Results files should have the following format and be stored as json files:
-```json
-{
-    "config": {
-        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
-        "model_name": "path of the model on the hub: org/model",
-        "model_sha": "revision on the hub",
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score,
-        },
-        "task_name2": {
-            "metric_name": score,
-        }
-    }
-}
+Interactive Gradio viewer for the
+[`JacobPEvans/mlx-benchmarks`](https://huggingface.co/datasets/JacobPEvans/mlx-benchmarks)
+dataset — a collection of benchmark runs of MLX-quantized and locally-hosted
+LLMs on Apple Silicon, all serialized under the envelope v1 schema.
+
+## Installation
+
+Requires Python 3.11+.
+
+```sh
+cd space
+python -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+## Usage
+
+Launch the viewer locally:
+
+```sh
+python app.py
+```
+
+Gradio prints a local URL (default `http://127.0.0.1:7860`). The app pulls all
+`data/*.parquet` shards from the HF dataset, caches them for 10 minutes, and
+offers three tabs:
+
+- **Bar chart — latest run** — one bar per model for a given (suite, task, metric)
+- **Trend — over time** — score trajectory per model across runs
+- **Summary table** — pivot of models x tasks for the selected (suite, metric)
+
+Hit **Refresh data** to invalidate the cache manually.
+
+## Deployment
+
+This Space is synced automatically from the
+[JacobPEvans/mlx-benchmarks](https://github.com/JacobPEvans/mlx-benchmarks)
+GitHub repository via the `deploy-space.yml` workflow on every `main` push
+that touches `space/`.
+
+## Contributing
+
+Issues and PRs live in the upstream
+[GitHub repo](https://github.com/JacobPEvans/mlx-benchmarks). See the
+repository `CONTRIBUTING.md` for the full developer workflow.
+
+## API
+
+The viewer is a Gradio UI — it has no stable programmatic API. To query the
+underlying data yourself, read the HF dataset directly:
+
+```python
+import pandas as pd
+from huggingface_hub import HfFileSystem
+
+fs = HfFileSystem()
+paths = fs.glob("datasets/JacobPEvans/mlx-benchmarks/data/*.parquet")
+df = pd.concat([pd.read_parquet(f"hf://{p}") for p in paths], ignore_index=True)
 ```
 
-Request files are created automatically by this tool.
+## License
 
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+Apache-2.0. See [LICENSE](https://github.com/JacobPEvans/mlx-benchmarks/blob/main/LICENSE).
 
-# Code logic for more complex edits
+## Source
 
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+Full source, tests, and developer docs:
+<https://github.com/JacobPEvans/mlx-benchmarks>

app.py

@@ -1,204 +1,279 @@
+"""
+MLX Benchmarks Viewer — Gradio Space
+
+Reads all parquet shards from JacobPEvans/mlx-benchmarks and renders
+interactive comparison charts. Auto-refreshes data every 10 minutes.
+
+Deploy to HF Spaces (SDK: gradio, Python 3.11+).
+"""
+
+import re
+import time
+from threading import Lock
+
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+import plotly.express as px
+import plotly.graph_objects as go
+from huggingface_hub import HfFileSystem
+
+DATASET = "datasets/JacobPEvans/mlx-benchmarks"
+CACHE_TTL = 600  # seconds
+EXPECTED_COLUMNS = ["timestamp", "suite", "name", "metric", "model", "value"]
+CSS = """
+#title { text-align: center; margin-bottom: 4px; }
+#subtitle { text-align: center; color: #666; margin-bottom: 20px; }
+"""
+
+
+# ── Data loading ──────────────────────────────────────────────────────────────
+
+_cache: tuple[float, pd.DataFrame] | None = None
+_cache_lock = Lock()
+
+
+def empty_data() -> pd.DataFrame:
+    return pd.DataFrame(columns=[*EXPECTED_COLUMNS, "model_short"])
+
+
+def load_data() -> pd.DataFrame:
+    global _cache
+    with _cache_lock:
+        if _cache and time.time() - _cache[0] < CACHE_TTL:
+            return _cache[1]
+
+        fs = HfFileSystem()
+        try:
+            paths = sorted(f"hf://{p}" for p in fs.glob(f"{DATASET}/data/*.parquet"))
+        except (FileNotFoundError, OSError):
+            paths = []
+
+        if not paths:
+            df = empty_data()
+            _cache = (time.time(), df)
+            return df
+
+        df = pd.concat([pd.read_parquet(p) for p in paths], ignore_index=True)
+        df["timestamp"] = pd.to_datetime(df["timestamp"], utc=True)
+        df["model_short"] = df["model"].apply(short_model)
+        _cache = (time.time(), df)
+        return df
+
+
+def short_model(name: str) -> str:
+    """Strip common prefixes for axis labels."""
+    name = re.sub(r"^mlx-community/", "", name)
+    name = re.sub(r"^openrouter/openai/", "openrouter/", name)
+    return name
+
+
+# ── Chart builders ────────────────────────────────────────────────────────────
+
+
+def bar_chart(df: pd.DataFrame, suite: str, task: str, metric: str) -> go.Figure:
+    """Latest-run bar chart: one bar per model, sorted by score."""
+    sub = df[(df["suite"] == suite) & (df["name"] == task) & (df["metric"] == metric)].copy()
+    if sub.empty:
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No data for this selection",
+            xref="paper",
+            yref="paper",
+            x=0.5,
+            y=0.5,
+            showarrow=False,
+            font_size=18,
+        )
+        return fig
+
+    # Keep only the latest run per model
+    sub = sub.sort_values("timestamp").groupby("model", as_index=False).last()
+    sub["label"] = sub["model_short"]
+    sub = sub.sort_values("value", ascending=True)
+    value_max = sub["value"].max()
+    axis_max = max(1.0, float(value_max) * 1.15) if pd.notna(value_max) else 1.0
+
+    fig = px.bar(
+        sub,
+        x="value",
+        y="label",
+        orientation="h",
+        text=sub["value"].map("{:.3f}".format),
+        color="value",
+        color_continuous_scale="Blues",
+        labels={"value": metric, "label": "Model"},
+        title=f"{task} — {metric}  ({suite})",
     )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    fig.update_traces(textposition="outside")
+    fig.update_coloraxes(showscale=False)
+    fig.update_layout(
+        height=max(350, len(sub) * 44),
+        margin={"l": 220, "r": 60, "t": 60, "b": 40},
+        yaxis_title="",
+        xaxis_range=[0, axis_max],
+        font_size=13,
     )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
+    return fig
+
+
+def trend_chart(df: pd.DataFrame, suite: str, task: str, metric: str, models: list[str]) -> go.Figure:
+    """Score-over-time line chart for selected models."""
+    sub = df[
+        (df["suite"] == suite)
+        & (df["name"] == task)
+        & (df["metric"] == metric)
+        & (df["model_short"].isin(models))
+    ].copy()
+    if sub.empty:
+        fig = go.Figure()
+        fig.add_annotation(
+            text="No data for this selection",
+            xref="paper",
+            yref="paper",
+            x=0.5,
+            y=0.5,
+            showarrow=False,
+            font_size=18,
+        )
+        return fig
+
+    fig = px.line(
+        sub.sort_values("timestamp"),
+        x="timestamp",
+        y="value",
+        color="model_short",
+        markers=True,
+        labels={"value": metric, "timestamp": "Run time", "model_short": "Model"},
+        title=f"{task} — {metric} over time",
     )
+    fig.update_layout(height=420, font_size=13, legend_title="")
+    return fig
+
+
+def summary_table(df: pd.DataFrame, suite: str, metric: str) -> pd.DataFrame:
+    """Pivot table: models x tasks, latest run only."""
+    sub = df[(df["suite"] == suite) & (df["metric"] == metric)].copy()
+    if sub.empty:
+        return pd.DataFrame({"(no data)": []})
+
+    sub = sub.sort_values("timestamp").groupby(["model", "name"], as_index=False).last()
+    pivot = sub.pivot(index="model_short", columns="name", values="value")
+    pivot = pivot.round(4).reset_index().rename(columns={"model_short": "Model"})
+    return pivot
+
+
+# ── Gradio UI ─────────────────────────────────────────────────────────────────
+
+
+def build_ui():
+    df = load_data()
+
+    suites = sorted(df["suite"].dropna().unique().tolist()) if not df.empty else ["reasoning"]
+    tasks = sorted(df["name"].dropna().unique().tolist()) if not df.empty else []
+    metrics = sorted(df["metric"].dropna().unique().tolist()) if not df.empty else []
+    models = sorted(df["model"].dropna().unique().tolist()) if not df.empty else []
+    model_labels = [short_model(m) for m in models]
+
+    default_suite = "reasoning" if "reasoning" in suites else (suites[0] if suites else "reasoning")
+    default_metric = (
+        "exact_match_flexible" if "exact_match_flexible" in metrics else (metrics[0] if metrics else None)
+    )
+
+    def filtered_tasks(suite):
+        d = load_data()
+        t = sorted(d[d["suite"] == suite]["name"].dropna().unique().tolist()) if not d.empty else []
+        return gr.Dropdown(choices=t, value=t[0] if t else None)
+
+    def filtered_metrics(suite):
+        d = load_data()
+        m = sorted(d[d["suite"] == suite]["metric"].dropna().unique().tolist()) if not d.empty else []
+        return gr.Dropdown(
+            choices=m, value="exact_match_flexible" if "exact_match_flexible" in m else (m[0] if m else None)
+        )
+
+    def update_bar(suite, task, metric):
+        return bar_chart(load_data(), suite, task, metric)
+
+    def update_trend(suite, task, metric, selected_models):
+        return trend_chart(load_data(), suite, task, metric, selected_models or model_labels)
+
+    def update_table(suite, metric):
+        return summary_table(load_data(), suite, metric)
+
+    def refresh():
+        global _cache
+        with _cache_lock:
+            _cache = None
+        d = load_data()
+        new_suites = sorted(d["suite"].dropna().unique().tolist()) if not d.empty else ["reasoning"]
+        new_tasks = sorted(d["name"].dropna().unique().tolist()) if not d.empty else []
+        new_metrics = sorted(d["metric"].dropna().unique().tolist()) if not d.empty else []
+        new_models = sorted(d["model"].dropna().unique().tolist()) if not d.empty else []
+        new_model_labels = [short_model(m) for m in new_models]
+        return (
+            gr.Dropdown(choices=new_suites, value=new_suites[0] if new_suites else None),
+            gr.Dropdown(choices=new_tasks, value=new_tasks[0] if new_tasks else None),
+            gr.Dropdown(choices=new_metrics, value=new_metrics[0] if new_metrics else None),
+            gr.CheckboxGroup(choices=new_model_labels, value=new_model_labels[:6]),
+            f"Loaded {len(d)} rows from {len(d['model'].unique()) if not d.empty else 0} models",
+        )
+
+    with gr.Blocks(title="MLX Benchmarks") as demo:
+        gr.Markdown("# MLX Benchmarks Viewer", elem_id="title")
+        gr.Markdown(
+            "Compare local MLX models and cloud endpoints across coding and reasoning benchmarks.  \n"
+            "Data: [JacobPEvans/mlx-benchmarks](https://huggingface.co/datasets/JacobPEvans/mlx-benchmarks)",
+            elem_id="subtitle",
+        )
+
+        with gr.Row():
+            suite_dd = gr.Dropdown(choices=suites, value=default_suite, label="Suite")
+            task_dd = gr.Dropdown(choices=tasks, value=tasks[0] if tasks else None, label="Task")
+            metric_dd = gr.Dropdown(choices=metrics, value=default_metric, label="Metric")
+            refresh_btn = gr.Button("↻ Refresh data", scale=0)
+
+        status = gr.Markdown(
+            f"Loaded {len(df)} rows from {df['model'].nunique() if not df.empty else 0} models."
+        )
+
+        with gr.Tabs():
+            with gr.Tab("Bar chart — latest run"):
+                bar_plot = gr.Plot(
+                    value=bar_chart(df, default_suite, tasks[0] if tasks else "", default_metric)
+                )
+
+            with gr.Tab("Trend — over time"):
+                model_select = gr.CheckboxGroup(
+                    choices=model_labels,
+                    value=model_labels[:6],
+                    label="Models to show",
+                )
+                trend_plot = gr.Plot()
+
+            with gr.Tab("Summary table"):
+                table_out = gr.DataFrame(
+                    value=summary_table(df, default_suite, default_metric),
+                    interactive=False,
+                )
+
+        # Wire up events
+        suite_dd.change(filtered_tasks, [suite_dd], [task_dd])
+        suite_dd.change(filtered_metrics, [suite_dd], [metric_dd])
+
+        for inp in [suite_dd, task_dd, metric_dd]:
+            inp.change(update_bar, [suite_dd, task_dd, metric_dd], [bar_plot])
+            inp.change(update_table, [suite_dd, metric_dd], [table_out])
+
+        for inp in [suite_dd, task_dd, metric_dd, model_select]:
+            inp.change(update_trend, [suite_dd, task_dd, metric_dd, model_select], [trend_plot])
+
+        refresh_btn.click(
+            refresh,
+            outputs=[suite_dd, task_dd, metric_dd, model_select, status],
+        )
+
+    return demo
 
 
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
-
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+if __name__ == "__main__":
+    build_ui().launch(css=CSS)

requirements.txt

@@ -1,16 +1,18 @@
-APScheduler
-black
-datasets
-gradio
-gradio[oauth]
-gradio_leaderboard==0.0.13
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
-numpy
-pandas
-python-dateutil
-tqdm
-transformers
-tokenizers>=0.15.0
-sentencepiece
+gradio>=6.12.0
+pandas>=2.0
+plotly>=6.7.0
+huggingface-hub>=1.0.0
+# Minimum-version pins for direct + transitive deps that have CVEs in
+# older versions. Lower-bound only — HF Spaces resolves the actual
+# installed version. Tighten to exact pins if reproducibility becomes
+# a concern.
+#   pillow  9.5.0   → 3 CVEs fixed in 10.4.0 (CVE-2024-28219, CVE-2023-50447, CVE-2024-44537)
+#   pillow  10.4.0  → GHSA-cfh3-3jmp-rvhc + GHSA-whj4-6x5x-4v2j fixed in 12.2.0
+#   pyarrow 14      → PYSEC-2023-238 + PYSEC-2024-161 fixed in 17.0.0
+#   pyarrow 17.0.0  → PYSEC-2026-113 (CVSS 7.0 High) fixed in 23.0.1
+#   orjson  3.9.9   → GHSA-hx9q-6w63-j58v fixed in 3.11.6
+#   idna    3.9.0   → GHSA-65pc-fj4g-8rjx fixed in 3.15 (CVSS 6.9, transitive via requests/httpx)
+pyarrow>=24.0.0
+pillow>=12.2.0
+orjson>=3.11.6
+idna>=3.15