upload leaderboard

app.py

@@ -0,0 +1,62 @@
+import gradio as gr
+import os
+import json
+from pathlib import Path
+
+from serve.css import block_css
+from serve.markdown import *
+from serve.leaderboard import build_leaderboard_tab
+from serve.model import model_config
+# SERVER_PORT, ROOT_PATH, ELO_RESULTS_DIR
+SERVER_PORT = 7860
+LEADERBOARD_DIR = "leaderboards"
+
+def build_combine_demo():
+    with gr.Blocks(
+        title="",
+        theme=gr.themes.Default(),
+        css=block_css,
+    ) as demo:
+        gr.Markdown(get_title_md(), elem_id="title_markdown")
+        gr.Markdown(get_intro_md(), elem_id="intro_markdown")
+
+        gr.Markdown("Models", elem_id="subtitle_markdown")
+        gr.Markdown(get_model_intro_md(), elem_id="intro_markdown")
+        with gr.Accordion("🔍 Expand to see detailed generative models involved", open=False):
+            gr.Markdown(get_model_description_md(model_config), elem_id="model_description_markdown")
+
+        gr.Markdown("Criteria", elem_id="subtitle_markdown")
+        gr.Markdown(get_object_dimension_intro_md(), elem_id="intro_markdown")
+        with gr.Accordion("🔍 Expand to see detailed evaluation dimensions", open=False):
+            gr.Markdown(get_object_dimension_description_md(), elem_id="evaldim_markdown")
+        
+        gr.Markdown("Leaderboard", elem_id="subtitle_markdown")
+        # gr.Markdown(get_leaderboard_intro_md(), elem_id="intro_markdown")
+        with gr.Tab("Hi3DEval"):
+            gr.Markdown(get_hi3deval_intro_md(), elem_id="intro_markdown")
+            hi3deval_leaderboard_file = gr.Textbox(value=os.path.join(LEADERBOARD_DIR, "object_hi3deval.csv"), visible=False)
+            hi3deval_task_dropdown = gr.Dropdown(
+                label="Choose Task",
+                choices=["Full Leaderboard", "Text-to-3D only", "Image-to-3D only"],
+                value="Full Leaderboard",  # 默认值
+                interactive=True,  # 可交互
+                multiselect=False  # 单选
+            )
+            hi3deval_leaderboard_table = build_leaderboard_tab(hi3deval_leaderboard_file)
+            gr.Markdown(get_citation_md("hi3deval"), elem_id="ack_markdown")
+
+        # with gr.Tab("Image-to-3D Generation", id=1):
+        #     build_leaderboard_tab()
+
+        hi3deval_task_dropdown.change(
+            build_leaderboard_tab,
+            [hi3deval_leaderboard_file, hi3deval_task_dropdown],
+            hi3deval_leaderboard_table
+        )
+    return demo
+
+
+if __name__ == "__main__":
+    demo = build_combine_demo()
+    # demo.queue(max_size=20).launch(server_port=server_port, root_path=ROOT_PATH, debug=True)
+    demo.queue(max_size=20).launch(server_port=SERVER_PORT, debug=True)

leaderboards/object_hi3deval.csv

@@ -0,0 +1,22 @@
+Method,Task,Geometry Plausibility,Geometry Details,Texture Quality,Geo.-Tex. Coherence,Prompt-3D Alignment,Overall
+hunyuan3d-2.0,"Image-to-3D",6.2919,2.7215,2.7644,0.9876,3.4334,16.1988
+trellis,"Image-to-3D",5.8626,2.392,2.4693,0.9702,3.5048,15.1989
+spar3d,"Image-to-3D",5.7791,2.3031,2.4749,0.9601,3.4842,15.0014
+triposr,"Image-to-3D",5.2216,2.4225,2.3758,0.9562,3.3643,14.3404
+instantmesh,"Image-to-3D",5.4242,2.2252,2.3063,0.9587,3.363,14.2775
+crm,"Image-to-3D",4.745,2.2991,2.3777,0.9164,3.219,13.5572
+mvdream,"Text-to-3D",4.4064,2.742,2.8116,0.951,2.5879,13.4989
+unique3d,"Image-to-3D",4.9288,2.3233,1.9627,0.776,3.1989,13.1897
+openlrm,"Image-to-3D",3.7754,2.2614,2.0922,0.902,2.2298,11.2608
+wonder3d,"Image-to-3D",3.7879,2.0092,1.9658,0.9255,2.0874,10.7758
+sz123,"Image-to-3D",3.6052,1.6548,2.0293,0.8902,2.2578,10.4374
+magic123,"Image-to-3D",3.4617,1.74,2.0094,0.898,2.2171,10.3262
+grm-i,"Image-to-3D",3.2932,1.857,1.8885,0.849,2.0735,9.9612
+lgm,"Image-to-3D",3.2148,1.6733,1.8891,0.8118,2.0304,9.6193
+lucid-dreamer,"Text-to-3D",2.9346,1.5891,2.1069,0.8333,2.0297,9.4936
+grm-t,"Text-to-3D",3.0096,1.6627,1.7389,0.898,1.793,9.1023
+latent-nerf,"Text-to-3D",2.7265,1.7067,1.7065,0.8412,1.7688,8.7497
+magic3d,"Text-to-3D",2.9015,1.6239,1.5395,0.9431,1.5618,8.5698
+syncdreamer,"Image-to-3D",2.9423,1.5323,1.2134,0.8529,1.2776,7.8185
+dreamfusion,"Text-to-3D",2.669,1.2525,1.183,0.9137,1.3446,7.3627
+triplane-gaussian,"Image-to-3D",2.2948,1.1859,1.2028,0.6647,1.2908,6.6389

requirements.txt

@@ -0,0 +1,70 @@
+aiofiles==23.2.1
+altair==5.5.0
+annotated-types==0.7.0
+anyio==4.9.0
+attrs==25.3.0
+Brotli==1.1.0
+certifi==2025.7.14
+charset-normalizer==3.4.2
+click==8.2.1
+contourpy==1.3.2
+cycler==0.12.1
+exceptiongroup==1.3.0
+fastapi==0.116.1
+ffmpy==0.6.1
+filelock==3.18.0
+fonttools==4.59.0
+fsspec==2025.7.0
+gradio==4.44.1
+gradio_client==1.3.0
+groovy==0.1.2
+h11==0.16.0
+hf-xet==1.1.5
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.34.1
+idna==3.10
+importlib_resources==6.5.2
+Jinja2==3.1.6
+jsonschema==4.25.0
+jsonschema-specifications==2025.4.1
+kiwisolver==1.4.8
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.10.3
+mdurl==0.1.2
+narwhals==1.48.1
+numpy==1.26.0
+orjson==3.11.1
+packaging==25.0
+pandas==2.3.1
+pillow==10.4.0
+pydantic==2.11.7
+pydantic_core==2.33.2
+pydub==0.25.1
+Pygments==2.19.2
+pyparsing==3.2.3
+python-dateutil==2.9.0.post0
+python-multipart==0.0.20
+pytz==2025.2
+PyYAML==6.0.2
+referencing==0.36.2
+requests==2.32.4
+rich==14.1.0
+rpds-py==0.26.0
+ruff==0.12.5
+safehttpx==0.1.6
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+starlette==0.47.2
+tomlkit==0.12.0
+tqdm==4.67.1
+typer==0.16.0
+typing-inspection==0.4.1
+typing_extensions==4.14.1
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.35.0
+websockets==11.0.3

serve/css.py

@@ -0,0 +1,66 @@
+block_css = """
+#title_markdown * {
+    font-size: 125% !important;
+    font-weight: bold;
+    text-align: center;
+    margin-bottom: 12px;
+}
+#subtitle_markdown * {
+    font-size: 110% !important;
+    font-weight: bold;
+    text-align: left;
+    margin-bottom: 2px;
+}
+#intro_markdown * {
+    font-size: 102% !important;
+    line-height: 1.2;
+}
+#model_description_markdown * {
+    font-size: 102% !important;
+}
+#model_description_markdown th {
+    display: none;
+}
+#model_description_markdown tr {
+    border: none !important;
+}
+#model_description_markdown table,
+#model_description_markdown td {
+    font-size: 100% !important;
+    border: none !important;
+    outline: none !important;
+    box-shadow: none !important;
+    background: none !important;
+    padding-top: 6px;
+    padding-bottom: 6px;
+}
+#evaldim_markdown * {
+    font-size: 102% !important;
+    line-height: 1.4;
+}
+#about_markdown {
+    font-size: 110%
+}
+#ack_markdown {
+    font-size: 100%
+}
+
+#leaderboard_markdown {
+    font-size: 110%
+}
+#leaderboard_markdown td {
+    padding-top: 6px;
+    padding-bottom: 6px;
+}
+#leaderboard_dataframe td {
+    line-height: 0.1em;
+    font-weight: bold;
+}
+#input_box textarea {
+    font-weight: bold;
+    font-size: 125%;
+}
+footer {
+    display:none !important
+}
+"""

serve/leaderboard.py

@@ -0,0 +1,96 @@
+import argparse
+import ast
+import pickle
+import os
+import threading
+import time
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+from serve.model import model_config
+
+def hyperlink(name, link):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); \
+    text-decoration: underline;text-decoration-style: dotted;">{name}</a>'
+
+def amend_model_name(name, rank):
+    model_name = model_config[name].model_name
+    if rank==1:
+        return "🥇 " + model_name
+    elif rank==2:
+        return "🥈 " + model_name
+    elif rank==3:
+        return '🥉 ' + model_name
+    else:
+        return model_name
+
+def get_cfg_info(name):
+    config = model_config[name]
+    links = []
+    if config.page_link:
+        links.append(hyperlink("Page", config.page_link))
+    if config.code_link:
+        links.append(hyperlink("Code", config.code_link))
+    return ", ".join(links) if links else "N/A", config.organization if config.organization else "N/A"
+
+def get_leaderboard_values(leaderboard_df):
+    leaderboard_vals = []
+    for i, row in leaderboard_df.iterrows():
+        rank = i+1
+        model_name = row["Method"]
+        task = row["Task"]
+        if model_name not in model_config.keys() or model_config[model_name].task != task:
+            continue
+        
+        values = [rank, amend_model_name(model_name, rank), task]
+        values = values + [row.get(dim, np.NaN) for dim in leaderboard_df.columns[2:]]
+        # values.append(round(np.sum([v for v in values[3:] if pd.notna(v)]), 4))
+
+        links, organization = get_cfg_info(model_name)
+        # values.append(links)
+        values.append(organization)
+        
+        leaderboard_vals.append(values)
+    return leaderboard_vals
+
+def get_topk_ranks(df, k=3):
+    ranks = {}
+    for col_idx, col in enumerate(df.columns[2:]):  # skip "Model" “Task"
+        topk = df[col].nlargest(k)
+        for rank, idx in enumerate(topk.index):
+            if idx not in ranks:
+                ranks[idx] = {}
+            ranks[idx][col_idx] = rank + 1  # 1-based rank
+    for i in range(k): ranks[i][5] = i + 1
+    return ranks  # dict: row -> {col: rank}
+
+def build_leaderboard_tab(leaderboard_file: str, task: str = ""):
+    if not isinstance(leaderboard_file, str):
+        leaderboard_file = leaderboard_file.value
+    if not isinstance(task, str):
+        task = task.value
+
+    df = pd.read_csv(leaderboard_file)
+    if task in ["Text-to-3D only", "Image-to-3D only"]:
+        df = df[df["Task"] == task.split()[0]]
+        # df = df.drop(df[df["Task"]!=task.split()[0]].index)
+    leaderboard_df = df.drop(df[df["Method"].isnull()].index)
+    leaderboard_df = leaderboard_df.reset_index(drop=True)
+
+    leaderboard_vals = get_leaderboard_values(leaderboard_df)
+    leaderboard = gr.Dataframe(
+        headers = ['Rank', "🤖 Model", "🪧 Task" ] 
+            + [f"{dim}" for dim in leaderboard_df.keys()[2:-1]] 
+            + ["⭐ Overall", "🏛️ Orgnization"],     # "🔗 Links",
+        datatype = ["number", "str", "str"] 
+            + ["number"] * (len(leaderboard_df.columns) - 3) 
+            + ["number", "str"],
+        value = leaderboard_vals,
+        height = 680,
+        column_widths = [60, 140, 100] 
+            + [120] * (len(leaderboard_df.columns) - 3) 
+            + [120, 160],
+        wrap = True,
+    )
+    return leaderboard

serve/markdown.py

@@ -0,0 +1,125 @@
+
+def hyperlink(name, link):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); \
+    text-decoration: underline;text-decoration-style: dotted;">{name}</a>'
+
+def get_title_md():
+    md = f'''
+#  🏆 Leaderboard for 3D Generative Models
+'''
+    return md
+
+def get_intro_md():
+    md = '''
+This leaderboard provides a centralized evaluation platform for evaluating and tracking the performance of 3D generation models.
+'''
+    return md
+
+def get_model_intro_md():
+    md = '''
+This leaderboard spans a diverse set of state-of-the-art 3D generation models, including different conditional settings such as images, text, or combinations thereof.
+'''
+    return md
+
+def get_model_description_md(model_config, cols=10, except_models=[]):
+    model_list = {}
+    for cfg in model_config.values():
+        task = cfg.task
+        model_name = cfg.model_name
+        model_link = cfg.page_link if cfg.page_link else cfg.code_link
+        if task not in model_list.keys():
+            model_list[task] = set()
+        if model_name not in except_models:
+            model_list[task].add(hyperlink(model_name, model_link))
+    
+    model_descriptions = ""
+    for task, models in model_list.items():
+        model_descriptions += f"\n**{len(models)} {task} Generative Models**\n"
+        ## model_table
+
+        model_descriptions += '<table style="width:100%; text-align:left; border:none; border-collapse: collapse;">\n'
+        for i, model in enumerate(models):
+            if i%cols == 0:
+                model_descriptions += ' <tr>\n'
+            model_descriptions += f'  <td>{model}</td>\n'
+            if (i+1)%cols == 0:
+                model_descriptions += ' </tr>\n'
+        if len(models)%cols != 0:
+            num_pad = cols - len(models)%cols
+            model_descriptions += '  <td></td>\n' * num_pad
+            model_descriptions += ' </tr>\n'
+        model_descriptions += '</table>\n'
+    return model_descriptions.strip()
+
+def get_object_dimension_intro_md():
+    md = f'''
+Each model involved is conducted under consistent and standardized settings and assessed along **multiple evaluation dimensions** to provide a detailed view of its strengths and limitations:
+'''
+    return md
+
+def get_object_dimension_description_md():
+    md = f'''
+- **Geometry Plausibility** assesses the structural integrity and physical feasibility of the generated shape.
+- **Geometry Details** reflects the fidelity of fine-scale structures, such as sharp edges and part boundaries. 
+- **Texture Quality** evaluates the visual fidelity of surface textures in terms of  resolution, realism, and aesthetic consistency.
+- **Geometry-Texture Coherency** assesses the alignment between texture and shape—whether textures follow the contours, part boundaries, and material semantics of geometry.
+- **Prompt-3D Alignment** evaluates the semantic and/or identity consistency between the input prompt and the generated 3D asset.
+'''
+    return md
+
+def get_leaderboard_intro_md():
+    md = '''
+This leaderboard integrates results from three complementary benchmarks that span different aspects of 3D synthesis. 
+- [Hi3DEval]()
+- [3DGenBench](https://zyh482.github.io/3DGen-Bench/)
+- [GPTEval3D](https://github.com/3DTopia/GPTEval3D)
+'''
+    return md
+
+
+def get_hi3deval_intro_md(num_model=None):
+    md = f'''
+This leaderboard is evaluated using **Hi3DEval**, a straight forward scoring benchmark that does **not rely on pairwise comparisons**.
+
+Specifically, each dimension is assigned an absolute score within clearly defined value ranges:
+
+- Geometry Plausibility: range [0, 9]
+- Geometry Details: range [0, 4]
+- Texture Quality: range [0, 4]
+- Geometry-Texture Coherency: range [0, 1]
+- Prompt-3D Alignment: range [0, 4]
+
+The **Overall Score** is computed as the **SUM** of the scores across all five dimensions.
+
+Hi3DEval supports unified evaluation for both **Text-to-3D** and **Image-to-3D** generation tasks. You can also freely select **"Task"** to explore performance under different input modalities.
+'''
+    return md
+
+def get_citation_md(name):
+    citations = {
+        "hi3deval": '''
+```bibtex
+@article
+        ''',
+        "3dgen-bench": '''
+```bibtex
+@article{zhang20253dgen,
+  title={3DGen-Bench: Comprehensive Benchmark Suite for 3D Generative Models},
+  author={Zhang, Yuhan and Zhang, Mengchen and Wu, Tong and Wang, Tengfei and Wetzstein, Gordon and Lin, Dahua and Liu, Ziwei},
+  journal={arXiv preprint arXiv:2503.21745},
+  year={2025}
+}
+        ''',
+        "gpteval3d": '''
+```bibtex
+@inproceedings{wu2024gpt,
+  title={Gpt-4v (ision) is a human-aligned evaluator for text-to-3d generation},
+  author={Wu, Tong and Yang, Guandao and Li, Zhibing and Zhang, Kai and Liu, Ziwei and Guibas, Leonidas and Lin, Dahua and Wetzstein, Gordon},
+  booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
+  pages={22227--22238},
+  year={2024}
+}
+        '''
+    }
+    md = f"Reference:\n{citations[name.lower()]}"
+    return md

serve/model.py

@@ -0,0 +1,353 @@
+from collections import namedtuple
+from typing import List
+
+ModelConfig = namedtuple("ModelConfig", ["model_name", "task", "representation", "paradigm", "page_link", "code_link", "organization"])
+model_config = {}
+
+def register_model_config(
+    nick_name:str, model_name: str, task: str, representation: str, paradigm: str = None, page_link: str = None, code_link: str = None, organization: str = None
+):
+    config = ModelConfig(model_name, task, representation, paradigm, page_link, code_link, organization)
+    model_config[nick_name] = config
+
+def get_model_config(model_name: str) -> ModelConfig:
+    assert model_name in model_config
+    return model_config[model_name]
+
+
+### Registering model configurations for TExt-to-3D models
+
+register_model_config(
+    nick_name="dreamfusion",
+    model_name="DreamFusion",
+    task="Text-to-3D",
+    representation="NeRF",
+    paradigm="Optimazation",
+    page_link="https://dreamfusion3d.github.io/",
+    code_link="",
+    organization="Google Research"
+)
+
+register_model_config(
+    nick_name="sjc",
+    model_name="SJC",
+    task="Text-to-3D",
+    representation="Voxel NeRF",  
+    paradigm="Optimization",
+    page_link="https://pals.ttic.edu/p/score-jacobian-chaining",
+    code_link="https://github.com/pals-ttic/sjc",
+    organization="TTI-Chicago"
+)
+
+register_model_config(
+    nick_name="latent-nerf",
+    model_name="Latent-NeRF",
+    task="Text-to-3D",
+    representation="NeRF",
+    paradigm="Optimization",
+    page_link="",
+    code_link="https://github.com/eladrich/latent-nerf",
+    organization="Tel Aviv University"
+)
+
+register_model_config(
+    nick_name="magic3d",
+    model_name="Magic3D",
+    task="Text-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://research.nvidia.com/labs/dir/magic3d/",
+    code_link="",
+    organization="Nvidia"
+)
+
+register_model_config(
+    nick_name="lucid-dreamer",
+    model_name="LucidDreamer",
+    task="Text-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://luciddreamer-cvlab.github.io/",
+    code_link="https://github.com/luciddreamer-cvlab/LucidDreamer",
+    organization="Computer Vision Lab, Seoul National University  "
+)
+
+register_model_config(
+    nick_name="mvdream",
+    model_name="MVDream",
+    task="Text-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://mv-dream.github.io/",
+    code_link="https://github.com/bytedance/MVDream",
+    organization="ByteDance"
+)
+
+register_model_config(
+    nick_name="grm-t",
+    model_name="GRM",
+    task="Text-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://justimyhxu.github.io/projects/grm/",
+    code_link="https://github.com/justimyhxu/grm",
+    organization="Stanford Univerity"
+)
+
+register_model_config(
+    nick_name="point-e-t",
+    model_name="Point-E",
+    task="Text-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://openai.com/index/point-e/",
+    code_link="https://github.com/openai/point-e",
+    organization="OpenAI"
+)
+
+register_model_config(
+    nick_name="shap-e-t",
+    model_name="Shap-E",
+    task="Text-to-3D",
+    representation="",
+    paradigm="",
+    page_link="",
+    code_link="https://github.com/openai/shap-e",
+    organization="OpenAI"
+)
+
+
+## Registering model configurations for Image-to-3D models
+register_model_config(
+    nick_name="trellis",
+    model_name="TRELLIS",
+    task="Image-to-3D",
+    representation="",
+    paradigm="Naive 3DGen",
+    page_link="https://microsoft.github.io/TRELLIS/",
+    code_link="https://github.com/Microsoft/TRELLIS",
+    organization="Microsoft Research"
+)
+register_model_config(
+    nick_name="hunyuan3d-2.0",
+    model_name="Hunyuan3D 2.0",
+    task="Image-to-3D",
+    representation="Mesh",
+    paradigm="Naive 3DGen",
+    page_link="https://3d-models.hunyuan.tencent.com/",
+    code_link="https://github.com/Tencent-Hunyuan/Hunyuan3D-2",
+    organization="Tencent Hunyuan3D Team"
+)
+register_model_config(
+    nick_name="hunyuan3d-2.5",
+    model_name="Hunyuan3D 2.5",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="",
+    code_link="",
+    organization="Tencent Hunyuan3D Team"
+)
+register_model_config(
+    nick_name="spar3d",
+    model_name="SPAR3D",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://spar3d.github.io/",
+    code_link="https://github.com/Stability-AI/stable-point-aware-3d",
+    organization="Stability AI"
+)
+register_model_config(
+    nick_name="instantmesh",
+    model_name="InstantMesh",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="",
+    code_link="https://github.com/TencentARC/InstantMesh",
+    organization="ARC Lab, Tencent PCG"
+)
+register_model_config(
+    nick_name="triposr",
+    model_name="TripoSR",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="",
+    code_link="https://github.com/VAST-AI-Research/TripoSR",
+    organization="Tripo AI, Staility AI"
+)
+register_model_config(
+    nick_name="unique3d",
+    model_name="Unique3D",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://wukailu.github.io/Unique3D/",
+    code_link="https://github.com/AiuniAI/Unique3D",
+    organization="Tsinghua University"
+)
+register_model_config(
+    nick_name="crm",
+    model_name="CRM",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://ml.cs.tsinghua.edu.cn/~zhengyi/CRM/",
+    code_link="https://github.com/thu-ml/CRM",
+    organization="Tsinghua University"
+)
+register_model_config(
+    nick_name="ln3diff",
+    model_name="LN3Diff",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://nirvanalan.github.io/projects/ln3diff/",
+    code_link="https://github.com/NIRVANALAN/LN3Diff",
+    organization="S-Lab, Nanyang Technological University"
+)
+register_model_config(
+    nick_name="wonder3d",
+    model_name="Wonder3D",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://www.xxlong.site/Wonder3D/",
+    code_link="https://github.com/xxlong0/Wonder3D",
+    organization="The University of Hong Kong"
+)
+register_model_config(
+    nick_name="openlrm",
+    model_name="OpenLRM",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="",
+    code_link="https://github.com/3DTopia/OpenLRM",
+    organization="Shanghai AI Lab"
+)
+register_model_config(
+    nick_name="sz123",
+    model_name="Stable Zero123",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://stability.ai/stable-3d",
+    code_link="https://huggingface.co/stabilityai/stable-zero123",
+    organization="Stability AI"
+)
+register_model_config(
+    nick_name="z123",
+    model_name="Zero-1-to-3 XL",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://zero123.cs.columbia.edu/",
+    code_link="https://github.com/cvlab-columbia/zero123",
+    organization="Columbia University"
+)
+register_model_config(
+    nick_name="magic123",
+    model_name="Magic123",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://guochengqian.github.io/project/magic123/",
+    code_link="https://github.com/guochengqian/Magic123",
+    organization="KAUST"
+)
+register_model_config(
+    nick_name="lgm",
+    model_name="LGM",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://me.kiui.moe/lgm/",
+    code_link="https://github.com/3DTopia/LGM",
+    organization="Peking University"
+)
+register_model_config(
+    nick_name="grm-i",
+    model_name="GRM",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://justimyhxu.github.io/projects/grm/",
+    code_link="https://github.com/justimyhxu/grm",
+    organization="Stanford Univerity"
+)
+register_model_config(
+    nick_name="syncdreamer",
+    model_name="SyncDreamer",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://liuyuan-pal.github.io/SyncDreamer/",
+    code_link="https://github.com/liuyuan-pal/SyncDreamer",
+    organization="The University of Hong Kong, Tencent Games"
+)
+register_model_config(
+    nick_name="shap-e-i",
+    model_name="Shap-E",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="",
+    code_link="https://github.com/openai/shap-e",
+    organization="OpenAI"
+)
+register_model_config(
+    nick_name="point-e-i",
+    model_name="Point-E",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://openai.com/index/point-e/",
+    code_link="https://github.com/openai/point-e",
+    organization="OpenAI"
+)
+register_model_config(
+    nick_name="escher-net",
+    model_name="EscherNet",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://kxhit.github.io/EscherNet",
+    code_link="https://github.com/kxhit/EscherNet",
+    organization="Dyson Robotics Lab, Imperial College London"
+)
+register_model_config(
+    nick_name="free3d",
+    model_name="Free3D",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://chuanxiaz.com/free3d/",
+    code_link="https://github.com/lyndonzheng/Free3D",
+    organization="Visual Geometry Group, University of Oxford"
+)
+
+register_model_config(
+    nick_name="triplane-gaussian",
+    model_name="TriplaneGaussian",
+    task="Image-to-3D",
+    representation="",
+    paradigm="",
+    page_link="https://zouzx.github.io/TriplaneGaussian/",
+    code_link="https://github.com/VAST-AI-Research/TriplaneGaussian",
+    organization="BNRist, Tsinghua University, VAST"
+)
+
+# register_model_config(
+#     nick_name="",
+#     model_name="",
+#     task="",
+#     representation="",
+#     paradigm="",
+#     page_link="",
+#     code_link="",
+#     organization=""
+# )
+

serve/tmp.py

@@ -0,0 +1,379 @@
+
+"""
+Live monitor of the website statistics and leaderboard.
+Dependency:
+sudo apt install pkg-config libicu-dev
+pip install pytz gradio gdown plotly polyglot pyicu pycld2 tabulate
+"""
+
+import argparse
+import ast
+import pickle
+import os
+import threading
+import time
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+
+
+basic_component_values = [None] * 6
+leader_component_values = [None] * 5
+
+nam_dict = {
+    "dreamfusion": "DreamFusion",
+    "mvdream": "MVDream",
+    "lucid-dreamer": "LucidDreamer",
+    "magic3d": "Magic3D",
+    "grm-t": "GRM", "grm-i": "GRM", "grm": "GRM",
+    "latent-nerf": "Latent-NeRF",
+    "shap-e-t": "Shap-E", "shap-e-i": "Shap-E", "shap-e": "Shap-E",
+    "point-e-t": "Point-E", "point-e-i": "Point-E", "point-e": "Point-E",
+    "sjc": "SJC",
+    "wonder3d": "Wonder3D",
+    "openlrm": "OpenLRM",
+    "sz123": "Stable Zero123", "stable-zero123": "Stable Zero123",
+    "z123": "Zero123-XL", "zero123-xl": "Zero123-XL",
+    "magic123": "Magic123",
+    "lgm": "LGM",
+    "syncdreamer": "SyncDreamer",
+    "triplane-gaussian": "TriplaneGaussian",
+    "escher-net": "EscherNet",
+    "free3d": "Free3D",
+    "instant-mesh": "InstantMesh",
+}
+
+def replace_model_name(name, rank):
+    name = nam_dict[name]
+
+    if rank==0:
+        return "🥇 "+name
+    elif rank==1:
+        return "🥈 "+name
+    elif rank==2:
+        return '🥉 '+name
+    else:
+        return name
+
+# def make_leaderboard_md(elo_results):
+#     leaderboard_md = f"""
+# # 🏆 Chatbot Arena Leaderboard
+# | [Blog](https://lmsys.org/blog/2023-05-03-arena/) | [GitHub](https://github.com/lm-sys/FastChat) | [Paper](https://arxiv.org/abs/2306.05685) | [Dataset](https://github.com/lm-sys/FastChat/blob/main/docs/dataset_release.md) | [Twitter](https://twitter.com/lmsysorg) | [Discord](https://discord.gg/HSWAKCrnFx) |
+
+# This leaderboard is based on the following three benchmarks.
+# - [Chatbot Arena](https://lmsys.org/blog/2023-05-03-arena/) - a crowdsourced, randomized battle platform. We use 100K+ user votes to compute Elo ratings.
+# - [MT-Bench](https://arxiv.org/abs/2306.05685) - a set of challenging multi-turn questions. We use GPT-4 to grade the model responses.
+# - [MMLU](https://arxiv.org/abs/2009.03300) (5-shot) - a test to measure a model's multitask accuracy on 57 tasks.
+
+# 💻 Code: The Arena Elo ratings are computed by this [notebook]({notebook_url}). The MT-bench scores (single-answer grading on a scale of 10) are computed by [fastchat.llm_judge](https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge). The MMLU scores are mostly computed by [InstructEval](https://github.com/declare-lab/instruct-eval). Higher values are better for all benchmarks. Empty cells mean not available. Last updated: November, 2023.
+# """
+#     return leaderboard_md
+
+def make_leaderboard_md(elo_results):
+    leaderboard_md = f"""
+# 🏆 3DGen-Arena Leaderboard
+"""
+    return leaderboard_md
+
+
+def make_leaderboard_md_live(elo_results):
+    leaderboard_md = f"""
+# Leaderboard
+Last updated: {elo_results["last_updated_datetime"]}
+{elo_results["leaderboard_table"]}
+"""
+    return leaderboard_md
+
+
+def model_hyperlink(model_name, link):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def load_leaderboard_table_csv(filename, add_hyperlink=True):
+    df = pd.read_csv(filename)
+    df = df.drop(df[df["Key"].isnull()].index)
+    for col in df.columns:
+        if "Elo rating" in col:
+            # print(col, df[col], type(df[col]), df[col] is not np.NaN)
+            df[col] = df[col].apply(lambda x: int(x) if (x != "-" and pd.notna(x)) else np.NaN)
+        
+        if add_hyperlink and col == "Model":
+            df[col] = df.apply(lambda row: model_hyperlink(row[col], row["Link"]), axis=1)
+    return df
+
+
+
+def build_basic_stats_tab():
+    empty = "Loading ..."
+    basic_component_values[:] = [empty, None, empty, empty, empty, empty]
+
+    md0 = gr.Markdown(empty)
+    gr.Markdown("#### Figure 1: Number of model calls and votes")
+    plot_1 = gr.Plot(show_label=False)
+    with gr.Row():
+        with gr.Column():
+            md1 = gr.Markdown(empty)
+        with gr.Column():
+            md2 = gr.Markdown(empty)
+    with gr.Row():
+        with gr.Column():
+            md3 = gr.Markdown(empty)
+        with gr.Column():
+            md4 = gr.Markdown(empty)
+    return [md0, plot_1, md1, md2, md3, md4]
+
+
+def get_full_table(anony_arena_df, full_arena_df, model_table_df):
+    values = []
+    for i in range(len(model_table_df)):
+        row = []
+        model_key = model_table_df.iloc[i]["Key"]
+        model_name = model_table_df.iloc[i]["Model"]
+        # model display name
+        row.append(model_name)
+        if model_key in anony_arena_df.index:
+            idx = anony_arena_df.index.get_loc(model_key)
+            row.append(round(anony_arena_df.iloc[idx]["rating"]))
+        else:
+            row.append(np.nan)
+        if model_key in full_arena_df.index:
+            idx = full_arena_df.index.get_loc(model_key)
+            row.append(round(full_arena_df.iloc[idx]["rating"]))
+        else:
+            row.append(np.nan)
+        # row.append(model_table_df.iloc[i]["MT-bench (score)"])
+        # row.append(model_table_df.iloc[i]["Num Battles"])
+        # row.append(model_table_df.iloc[i]["MMLU"])
+        # Organization
+        row.append(model_table_df.iloc[i]["Organization"])
+        # license
+        row.append(model_table_df.iloc[i]["License"])
+
+        values.append(row)
+    values.sort(key=lambda x: -x[1] if not np.isnan(x[1]) else 1e9)
+    return values
+
+
+def get_arena_table(arena_dfs, model_table_df):
+    # sort by rating
+    # arena_df = arena_df.sort_values(by=["rating"], ascending=False)
+    values = []
+    for i in range(len(model_table_df)):
+        row = []
+        # model_key = arena_df.index[i]
+        # model_name = model_table_df[model_table_df["Key"] == model_key]["Model"].values[
+        #     0
+        # ]
+        model_name = model_table_df.iloc[i]["Key"]
+
+        # rank
+        row.append(i + 1)
+        # model display name
+        row.append(replace_model_name(model_name, i))
+        # elo rating
+        num_battles = 0
+        for dim in arena_dfs.keys():
+            # try:
+            #     print(arena_dfs[dim].loc[model_name])
+            # except:
+            #     continue
+            row.append(round(arena_dfs[dim].loc[model_name]["rating"], 2))
+            upper_diff = round(arena_dfs[dim].loc[model_name]["rating_q975"] - arena_dfs[dim].loc[model_name]["rating"])
+            lower_diff = round(arena_dfs[dim].loc[model_name]["rating"] - arena_dfs[dim].loc[model_name]["rating_q025"])
+            # row.append(f"+{upper_diff}/-{lower_diff}")
+            try:
+                num_battles += round(arena_dfs[dim].loc[model_name]["num_battles"])
+            except:
+                num_battles += 0
+        # row.append(round(arena_df.iloc[i]["rating"]))
+        # upper_diff = round(arena_df.iloc[i]["rating_q975"] - arena_df.iloc[i]["rating"])
+        # lower_diff = round(arena_df.iloc[i]["rating"] - arena_df.iloc[i]["rating_q025"])
+        # row.append(f"+{upper_diff}/-{lower_diff}")
+        row.append(round(model_table_df.iloc[i]["Arena Elo rating"], 2))
+        # num battles
+        # row.append(round(arena_df.iloc[i]["num_battles"]))
+        row.append(num_battles)
+        # Organization
+        # row.append(
+        #     model_table_df[model_table_df["Key"] == model_key]["Organization"].values[0]
+        # )
+        # # license
+        # row.append(
+        #     model_table_df[model_table_df["Key"] == model_key]["License"].values[0]
+        # )
+
+        values.append(row)
+    return values
+
+def make_arena_leaderboard_md(elo_results):
+    total_votes = 0
+    for dim in elo_results.keys():
+        arena_df = elo_results[dim]["leaderboard_table_df"]
+        last_updated = elo_results[dim]["last_updated_datetime"]
+        total_votes += sum(arena_df["num_battles"].fillna(0)) // 2
+        total_models = len(arena_df)
+
+    leaderboard_md = f"""
+Total #models: **{total_models}**. \n
+Total #votes: **{int(total_votes)}** (Anonymous Votes only). \n
+Last updated: {last_updated}. \n
+Contribute the votes 🗳️ at [3DGen-Arena](https://huggingface.co/spaces/ZhangYuhan/3DGen-Arena)! 
+"""
+    return leaderboard_md
+
+def make_full_leaderboard_md(elo_results):
+    total_votes = 0
+    for dim in elo_results.keys():
+        arena_df = elo_results[dim]["leaderboard_table_df"]
+        last_updated = elo_results[dim]["last_updated_datetime"]
+        total_votes += sum(arena_df["num_battles"].fillna(0)) // 2
+        total_models = len(arena_df)
+
+    leaderboard_md = f"""
+Total #models: **{total_models}**. \n
+Total #votes: **{int(total_votes)}** (Anonymous + Named Votes). \n
+Last updated: {last_updated}.\n
+Contribute the votes 🗳️ at [3DGen-Arena](https://huggingface.co/spaces/ZhangYuhan/3DGen-Arena)! 
+"""
+    return leaderboard_md
+
+def build_empty_leaderboard_tab():
+    leaderboard_md = f"""
+        # 🗳️ Leaderboard
+        ## Look forward to your votes, and the leaderboard is coming soon!
+    """
+    gr.Markdown(leaderboard_md, elem_id="leaderboard_markdown")
+
+def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=False):
+    if elo_results_file is None:  # Do live update
+        md = "Loading ..."
+        p1 = p2 = p3 = p4 = None
+    else:
+        with open(elo_results_file, "rb") as fin:
+            elo_results = pickle.load(fin)
+            
+        # print(elo_results)
+        # print(elo_results.keys())
+        anony_elo_results, full_elo_results = {}, {}
+        anony_arena_dfs, full_arena_dfs = {}, {}
+        p1, p2, p3, p4 = {}, {}, {}, {}
+        for dim in elo_results.keys():
+            anony_elo_results[dim] = elo_results[dim]["anony"]
+            full_elo_results[dim] = elo_results[dim]["full"]
+            anony_arena_dfs[dim] = anony_elo_results[dim]["leaderboard_table_df"]
+            full_arena_dfs[dim] = full_elo_results[dim]["leaderboard_table_df"]
+            p1[dim] = anony_elo_results[dim]["win_fraction_heatmap"]
+            p2[dim] = anony_elo_results[dim]["battle_count_heatmap"]
+            p3[dim] = anony_elo_results[dim]["bootstrap_elo_rating"]
+            p4[dim] = anony_elo_results[dim]["average_win_rate_bar"]
+            print(anony_arena_dfs[dim])
+            print(full_arena_dfs[dim])
+        
+        md = make_leaderboard_md(anony_elo_results)
+        
+    md_1 = gr.Markdown(md, elem_id="leaderboard_markdown")
+
+    if leaderboard_table_file:
+        model_table_df = load_leaderboard_table_csv(leaderboard_table_file)
+        model_table_df_full = load_leaderboard_table_csv(str(leaderboard_table_file).replace('.csv', '_full.csv'))
+        with gr.Tabs() as tabs:
+            # arena table
+            arena_table_vals = get_arena_table(anony_arena_dfs, model_table_df)
+            with gr.Tab("Anony. Arena", id=0):
+                md = make_arena_leaderboard_md(anony_elo_results)
+                gr.Markdown(md, elem_id="leaderboard_markdown")
+                gr.Dataframe(
+                    # headers=[
+                    #     "Rank",
+                    #     "🤖 Model",
+                    #     "⭐ Arena Elo",
+                    #     "📊 95% CI",
+                    #     "🗳️ Votes",
+                    #     "Organization",
+                    #     "License",
+                    # ],
+                    headers=["Rank", "🤖 Model"] + [f"📈 {dim} Elo" for dim in anony_arena_dfs.keys()] + ["⭐ Avg. Arena Elo Ranking", "📮 Votes"],
+                    datatype=[
+                        "str",
+                        "markdown",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number"
+                    ],
+                    value=arena_table_vals,
+                    # value=model_table_df,
+                    elem_id="arena_leaderboard_dataframe",
+                    height=700,
+                    column_widths=[50, 200, 100, 100, 100, 100, 100, 100, 100],
+                    wrap=True,
+                )
+            with gr.Tab("Full Arena", id=1):
+                md = make_full_leaderboard_md(full_elo_results)
+                gr.Markdown(md, elem_id="leaderboard_markdown")
+                full_table_vals = get_arena_table(full_arena_dfs, model_table_df_full)
+                gr.Dataframe(
+                    headers=["Rank", "🤖 Model"] + [f"📈 {dim} Elo" for dim in anony_arena_dfs.keys()] + ["⭐ Avg. Arena Elo Ranking", "📮 Votes"],
+                    datatype=[
+                        "str",
+                        "markdown",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number",
+                        "number"
+                    ],
+                    value=full_table_vals,
+                    elem_id="full_leaderboard_dataframe",
+                    column_widths=[50, 200, 100, 100, 100, 100, 100, 100, 100],
+                    height=700,
+                    wrap=True,
+                )
+        if not show_plot:
+            gr.Markdown(
+                """ ## We are still collecting more votes on more models. The ranking will be updated very fruquently. Please stay tuned! 
+                """,
+                elem_id="leaderboard_markdown",
+            )
+    else:
+        pass
+
+    # leader_component_values[:] = [md, p1, p2, p3, p4]
+
+    """
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown(
+                "#### Figure 1: Fraction of Model A Wins for All Non-tied A vs. B Battles"
+            )
+            plot_1 = gr.Plot(p1, show_label=False)
+        with gr.Column():
+            gr.Markdown(
+                "#### Figure 2: Battle Count for Each Combination of Models (without Ties)"
+            )
+            plot_2 = gr.Plot(p2, show_label=False)
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown(
+                "#### Figure 3: Bootstrap of Elo Estimates (1000 Rounds of Random Sampling)"
+            )
+            plot_3 = gr.Plot(p3, show_label=False)
+        with gr.Column():
+            gr.Markdown(
+                "#### Figure 4: Average Win Rate Against All Other Models (Assuming Uniform Sampling and No Ties)"
+            )
+            plot_4 = gr.Plot(p4, show_label=False)
+    """
+
+    from .utils import acknowledgment_md
+
+    gr.Markdown(acknowledgment_md)
+
+    # return [md_1, plot_1, plot_2, plot_3, plot_4]
+    return [md_1]