| """CADGenBench Leaderboard Space. |
| |
| Step 3 prototype: a hand-crafted ``results.jsonl`` drives the leaderboard |
| table, and the Submit tab is a UI-only stub. The read path (Step 5) will |
| swap the JSONL for ``datasets.load_dataset(HF_SUBMISSIONS_REPO, 'results')`` |
| and the write path (Step 6) will run ``cadgenbench evaluate`` and push a |
| result row back to the submissions dataset via ``HfApi``. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import json |
| import os |
| from pathlib import Path |
|
|
| import gradio as gr |
| import pandas as pd |
| from huggingface_hub import hf_hub_download |
|
|
| HF_ORG = os.getenv("HF_ORG", "michaelr27") |
| HF_SUBMISSIONS_REPO = os.getenv( |
| "HF_SUBMISSIONS_REPO", f"{HF_ORG}/cadgenbench-submissions" |
| ) |
| HF_DATA_REPO = os.getenv("HF_DATA_REPO", f"{HF_ORG}/cadgenbench-data") |
|
|
| LOCAL_RESULTS_PATH = Path(__file__).parent / "results.jsonl" |
|
|
| LEADERBOARD_COLS = [ |
| "model", |
| "submitter_name", |
| "aggregate_score", |
| "validity_rate", |
| "submitted_at", |
| "cadgenbench_version", |
| ] |
|
|
|
|
| def _load_rows_from_hub() -> list[dict] | None: |
| """Pull results.jsonl from the submissions dataset. |
| |
| Returns None on any failure so callers can fall back to the local file. |
| """ |
| try: |
| path = hf_hub_download( |
| repo_id=HF_SUBMISSIONS_REPO, |
| filename="results.jsonl", |
| repo_type="dataset", |
| force_download=True, |
| ) |
| return [ |
| json.loads(line) |
| for line in Path(path).read_text().splitlines() |
| if line.strip() |
| ] |
| except Exception as e: |
| print(f"[load_leaderboard] Hub fetch failed ({type(e).__name__}: {e})") |
| return None |
|
|
|
|
| def _load_rows_from_local() -> list[dict]: |
| if not LOCAL_RESULTS_PATH.exists(): |
| return [] |
| return [ |
| json.loads(line) |
| for line in LOCAL_RESULTS_PATH.read_text().splitlines() |
| if line.strip() |
| ] |
|
|
|
|
| def load_leaderboard() -> pd.DataFrame: |
| rows = _load_rows_from_hub() |
| if rows is None: |
| print("[load_leaderboard] falling back to local results.jsonl") |
| rows = _load_rows_from_local() |
| if not rows: |
| return pd.DataFrame(columns=LEADERBOARD_COLS) |
| df = pd.DataFrame(rows) |
| cols = [c for c in LEADERBOARD_COLS if c in df.columns] |
| return ( |
| df[cols] |
| .sort_values("aggregate_score", ascending=False, na_position="last") |
| .reset_index(drop=True) |
| ) |
|
|
|
|
| def handle_submit( |
| zip_file, |
| model: str, |
| submitter: str, |
| agent_url: str, |
| notes: str, |
| agree: bool, |
| ) -> str: |
| if zip_file is None: |
| return "**Error:** please attach a submission zip." |
| if not model.strip(): |
| return "**Error:** please fill in the Model identifier." |
| if not submitter.strip(): |
| return "**Error:** please fill in your Submitter name." |
| if not agree: |
| return "**Error:** you must agree to publish before submitting." |
|
|
| name = Path(zip_file.name).name |
| return ( |
| f"Received `{name}` for model `{model}` by `{submitter}`.\n\n" |
| f"_Evaluation is not wired yet (Step 6 of the build plan). Once it " |
| f"is, this submission will run the CPU eval inline and append a row " |
| f"to `{HF_SUBMISSIONS_REPO}`._" |
| ) |
|
|
|
|
| ABOUT_MD = f"""## About |
| |
| **CADGenBench** evaluates AI-driven CAD generation: how well a model can |
| turn a description of a mechanical part into a valid, geometrically |
| correct 3D model. |
| |
| - Reference baseline: an iterative AI agent that writes build123d Python. |
| - Submission flow: upload a zip of per-fixture STEP files; the Space runs |
| the CPU eval and appends a row to the submissions dataset. |
| - Datasets: fixtures (inputs + ground truth) live in `{HF_DATA_REPO}`; |
| submissions and computed results live in `{HF_SUBMISSIONS_REPO}`. |
| |
| ### Status |
| |
| This Space is in **active development** under `{HF_ORG}/AI4Engineering` and |
| will move to `science/cadgenbench-leaderboard` before going public. See |
| `space-setup/` in the source tree for the full build plan. |
| """ |
|
|
| with gr.Blocks(title="CADGenBench Leaderboard") as app: |
| gr.Markdown( |
| "# CADGenBench Leaderboard\n" |
| "_Benchmarking AI-driven CAD generation._" |
| ) |
|
|
| with gr.Tab("Leaderboard"): |
| df_view = gr.Dataframe( |
| value=load_leaderboard(), |
| interactive=False, |
| wrap=True, |
| label="Results (sorted by aggregate CAD score)", |
| ) |
| refresh_btn = gr.Button("Refresh", size="sm") |
| refresh_btn.click(fn=load_leaderboard, outputs=df_view) |
|
|
| with gr.Tab("Submit"): |
| gr.Markdown( |
| f""" |
| **Submission format.** A single zip with: |
| |
| - one folder per fixture in `{HF_DATA_REPO}`, each containing `output.step`; |
| - a top-level `meta.json`: |
| |
| ```json |
| {{ |
| "submitter_name": "your name or team", |
| "model": "anthropic/claude-sonnet-4-6", |
| "agent_url": "https://github.com/... (optional)", |
| "notes": "free text, optional, max 500 chars, single line, plain text", |
| "agree_to_publish": true |
| }} |
| ``` |
| |
| **Notes field.** Plain text only (no markdown / HTML). Capped at 500 chars |
| and stripped to a single line. Shown in the per-submission detail view, |
| not in the main leaderboard table. |
| |
| The Space runs the CPU eval inline and appends a row to |
| `{HF_SUBMISSIONS_REPO}`. You can fill the fields below to override |
| `meta.json` for a quick test. |
| """ |
| ) |
| zip_in = gr.File(label="Submission ZIP", file_types=[".zip"]) |
| with gr.Row(): |
| model_in = gr.Textbox( |
| label="Model identifier", |
| placeholder="e.g. anthropic/claude-sonnet-4-6", |
| ) |
| submitter_in = gr.Textbox(label="Submitter name") |
| with gr.Row(): |
| agent_url_in = gr.Textbox( |
| label="Agent / paper URL (optional)", |
| placeholder="https://github.com/...", |
| ) |
| notes_in = gr.Textbox(label="Notes (optional)") |
| agree_in = gr.Checkbox( |
| label="I agree to publish this result on the public leaderboard." |
| ) |
| submit_btn = gr.Button("Submit", variant="primary") |
| submit_out = gr.Markdown() |
| submit_btn.click( |
| fn=handle_submit, |
| inputs=[ |
| zip_in, |
| model_in, |
| submitter_in, |
| agent_url_in, |
| notes_in, |
| agree_in, |
| ], |
| outputs=submit_out, |
| ) |
|
|
| with gr.Tab("About"): |
| gr.Markdown(ABOUT_MD) |
|
|
|
|
| if __name__ == "__main__": |
| app.launch(theme=gr.themes.Soft()) |
|
|
