app.py

"""CADGenBench Leaderboard Space.

Step 3 prototype: a hand-crafted ``results.jsonl`` drives the leaderboard
table, and the Submit tab is a UI-only stub. The read path (Step 5) will
swap the JSONL for ``datasets.load_dataset(HF_SUBMISSIONS_REPO, 'results')``
and the write path (Step 6) will run ``cadgenbench evaluate`` and push a
result row back to the submissions dataset via ``HfApi``.
"""

from __future__ import annotations

import json
import os
from pathlib import Path

import gradio as gr
import pandas as pd
from huggingface_hub import hf_hub_download

HF_ORG = os.getenv("HF_ORG", "michaelr27")
HF_SUBMISSIONS_REPO = os.getenv(
    "HF_SUBMISSIONS_REPO", f"{HF_ORG}/cadgenbench-submissions"
)
HF_DATA_REPO = os.getenv("HF_DATA_REPO", f"{HF_ORG}/cadgenbench-data")

LOCAL_RESULTS_PATH = Path(__file__).parent / "results.jsonl"

LEADERBOARD_COLS = [
    "model",
    "submitter_name",
    "aggregate_score",
    "validity_rate",
    "submitted_at",
    "cadgenbench_version",
]


def _load_rows_from_hub() -> list[dict] | None:
    """Pull results.jsonl from the submissions dataset.

    Returns None on any failure so callers can fall back to the local file.
    """
    try:
        path = hf_hub_download(
            repo_id=HF_SUBMISSIONS_REPO,
            filename="results.jsonl",
            repo_type="dataset",
            force_download=True,
        )
        return [
            json.loads(line)
            for line in Path(path).read_text().splitlines()
            if line.strip()
        ]
    except Exception as e:  # noqa: BLE001 — any failure should fall back
        print(f"[load_leaderboard] Hub fetch failed ({type(e).__name__}: {e})")
        return None


def _load_rows_from_local() -> list[dict]:
    if not LOCAL_RESULTS_PATH.exists():
        return []
    return [
        json.loads(line)
        for line in LOCAL_RESULTS_PATH.read_text().splitlines()
        if line.strip()
    ]


def load_leaderboard() -> pd.DataFrame:
    rows = _load_rows_from_hub()
    if rows is None:
        print("[load_leaderboard] falling back to local results.jsonl")
        rows = _load_rows_from_local()
    if not rows:
        return pd.DataFrame(columns=LEADERBOARD_COLS)
    df = pd.DataFrame(rows)
    cols = [c for c in LEADERBOARD_COLS if c in df.columns]
    return (
        df[cols]
        .sort_values("aggregate_score", ascending=False, na_position="last")
        .reset_index(drop=True)
    )


def handle_submit(
    zip_file,
    model: str,
    submitter: str,
    agent_url: str,
    notes: str,
    agree: bool,
) -> str:
    if zip_file is None:
        return "**Error:** please attach a submission zip."
    if not model.strip():
        return "**Error:** please fill in the Model identifier."
    if not submitter.strip():
        return "**Error:** please fill in your Submitter name."
    if not agree:
        return "**Error:** you must agree to publish before submitting."

    name = Path(zip_file.name).name
    return (
        f"Received `{name}` for model `{model}` by `{submitter}`.\n\n"
        f"_Evaluation is not wired yet (Step 6 of the build plan). Once it "
        f"is, this submission will run the CPU eval inline and append a row "
        f"to `{HF_SUBMISSIONS_REPO}`._"
    )


ABOUT_MD = f"""## About

**CADGenBench** evaluates AI-driven CAD generation: how well a model can
turn a description of a mechanical part into a valid, geometrically
correct 3D model.

- Reference baseline: an iterative AI agent that writes build123d Python.
- Submission flow: upload a zip of per-fixture STEP files; the Space runs
  the CPU eval and appends a row to the submissions dataset.
- Datasets: fixtures (inputs + ground truth) live in `{HF_DATA_REPO}`;
  submissions and computed results live in `{HF_SUBMISSIONS_REPO}`.

### Status

This Space is in **active development** under `{HF_ORG}/AI4Engineering` and
will move to `science/cadgenbench-leaderboard` before going public. See
`space-setup/` in the source tree for the full build plan.
"""

with gr.Blocks(title="CADGenBench Leaderboard") as app:
    gr.Markdown(
        "# CADGenBench Leaderboard\n"
        "_Benchmarking AI-driven CAD generation._"
    )

    with gr.Tab("Leaderboard"):
        df_view = gr.Dataframe(
            value=load_leaderboard(),
            interactive=False,
            wrap=True,
            label="Results (sorted by aggregate CAD score)",
        )
        refresh_btn = gr.Button("Refresh", size="sm")
        refresh_btn.click(fn=load_leaderboard, outputs=df_view)

    with gr.Tab("Submit"):
        gr.Markdown(
            f"""
**Submission format.** A single zip with:

- one folder per fixture in `{HF_DATA_REPO}`, each containing `output.step`;
- a top-level `meta.json`:

```json
{{
  "submitter_name": "your name or team",
  "model": "anthropic/claude-sonnet-4-6",
  "agent_url": "https://github.com/...   (optional)",
  "notes": "free text, optional, max 500 chars, single line, plain text",
  "agree_to_publish": true
}}
```

**Notes field.** Plain text only (no markdown / HTML). Capped at 500 chars
and stripped to a single line. Shown in the per-submission detail view,
not in the main leaderboard table.

The Space runs the CPU eval inline and appends a row to
`{HF_SUBMISSIONS_REPO}`. You can fill the fields below to override
`meta.json` for a quick test.
"""
        )
        zip_in = gr.File(label="Submission ZIP", file_types=[".zip"])
        with gr.Row():
            model_in = gr.Textbox(
                label="Model identifier",
                placeholder="e.g. anthropic/claude-sonnet-4-6",
            )
            submitter_in = gr.Textbox(label="Submitter name")
        with gr.Row():
            agent_url_in = gr.Textbox(
                label="Agent / paper URL (optional)",
                placeholder="https://github.com/...",
            )
            notes_in = gr.Textbox(label="Notes (optional)")
        agree_in = gr.Checkbox(
            label="I agree to publish this result on the public leaderboard."
        )
        submit_btn = gr.Button("Submit", variant="primary")
        submit_out = gr.Markdown()
        submit_btn.click(
            fn=handle_submit,
            inputs=[
                zip_in,
                model_in,
                submitter_in,
                agent_url_in,
                notes_in,
                agree_in,
            ],
            outputs=submit_out,
        )

    with gr.Tab("About"):
        gr.Markdown(ABOUT_MD)


if __name__ == "__main__":
    app.launch(theme=gr.themes.Soft())