Update leaderboard sorting and readability

Add sorting by every leaderboard item and improve text contrast.

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text
+assets/avbench_outline.png filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Microsoft Corporation.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,48 +1,74 @@
 ---
-title: AVGen Bench
-emoji: 🥇
-colorFrom: green
-colorTo: indigo
+title: AVGen-Bench Leaderboard
 sdk: gradio
 app_file: app.py
-pinned: true
+pinned: false
 license: mit
-short_description: Duplicate this leaderboard to initialize your own!
-sdk_version: 5.43.1
-tags:
-- leaderboard
 ---
 
-# Start the configuration
-
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
-
-Results files should have the following format and be stored as json files:
-```json
-{
-    "config": {
-        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
-        "model_name": "path of the model on the hub: org/model",
-        "model_sha": "revision on the hub",
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score,
-        },
-        "task_name2": {
-            "metric_name": score,
-        }
-    }
-}
+# AVGen-Bench Leaderboard
+
+This is a Hugging Face Space for the AVGen-Bench Text-to-Audio-Video generation leaderboard.
+
+The app loads scores from `data/leaderboard.csv`, renders an interactive leaderboard, and documents the Scheme 2 aggregate metric used by AVGen-Bench.
+
+## Local Run
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+python app.py
+```
+
+## Deploy to Hugging Face Spaces
+
+```bash
+git init
+git branch -M main
+git add .
+git commit -m "Initial AVGen-Bench leaderboard Space"
+
+hf auth login
+hf repos create spaces/<your-username>/AVGen-Bench-Leaderboard --type space --space-sdk gradio
+git remote add space https://huggingface.co/spaces/<your-username>/AVGen-Bench-Leaderboard
+git push space main
+```
+
+## Updating Results
+
+Edit `data/leaderboard.csv` with one row per model. The expected columns are:
+
+```text
+Model, Components, Component Type, Vis, Aud (PQ), AV, Lip, Text, Face, Music,
+Speech, Lo-Phy, Hi-Phy, Holistic, Total, Source
+```
+
+`data/submission_template.csv` provides a one-row template for new submissions.
+
+## Public Submissions
+
+The Space includes a `Submission` tab and a Gradio API endpoint named `submit_score`.
+Submitted entries are treated as pending review:
+
+1. Users submit raw metric values, model metadata, a public contact, and evaluation artifact links.
+2. The app recomputes `Total` from the raw metrics using the AVGen-Bench Scheme 2 formula.
+3. The submission is written to a review backend; accepted entries should then be merged into `data/leaderboard.csv`.
+
+Production backend:
+
+```bash
+SUBMISSION_BACKEND=github_issue
+GITHUB_REPO=<owner>/<repo>
+GITHUB_TOKEN=<token-with-issues-write-access>
 ```
 
-Request files are created automatically by this tool.
+Store `GITHUB_TOKEN` as a Hugging Face Space secret, not in source control. If those variables are absent, the app falls back to `local_file` and writes JSON packets under `pending_submissions/`; for a production Space, use the GitHub issue backend or set `PENDING_SUBMISSION_DIR=/data/pending_submissions` with persistent Space storage.
 
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+## Sources
 
-# Code logic for more complex edits
+Leaderboard values and the overview figure were initialized from `microsoft/AVGen-Bench` commit `1049eab`.
 
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+- Project: https://github.com/microsoft/AVGen-Bench
+- Paper: https://arxiv.org/abs/2604.08540
+- Dataset: https://huggingface.co/datasets/microsoft/AVGen-Bench

app.py

@@ -1,204 +1,590 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
+
+from leaderboard import (
+    DATA_PATH,
+    SORT_CHOICES,
+    filter_leaderboard,
+    load_leaderboard,
+    metric_standings,
+    model_choices,
+    render_methodology,
+    render_profile,
+    render_summary,
+    render_table,
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+from submission import submit_score
 
 
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
+ROOT = Path(__file__).parent
+OVERVIEW_IMAGE = ROOT / "assets" / "avbench_outline.png"
+SUBMISSION_TEMPLATE = ROOT / "data" / "submission_template.csv"
 
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
+LEADERBOARD = load_leaderboard(DATA_PATH)
+STANDINGS = metric_standings(LEADERBOARD)
 
 
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+def update_leaderboard(component_type: str, query: str, sort_by: str, sort_order: str):
+    view = filter_leaderboard(LEADERBOARD, component_type, query, sort_by, sort_order)
+    return render_summary(LEADERBOARD, view), render_table(view, STANDINGS, sort_by, sort_order)
 
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
+
+def update_profile(model: str):
+    return render_profile(LEADERBOARD, model)
+
+
+CSS = """
+:root {
+  --avgen-bg: #f6f7f9;
+  --avgen-panel: #ffffff;
+  --avgen-ink: #18202b;
+  --avgen-muted: #5f6b7a;
+  --avgen-line: #d9dee7;
+  --avgen-blue: #1d5f9f;
+  --avgen-teal: #087a73;
+  --avgen-orange: #b45309;
+  --avgen-green: #24784f;
+}
+
+.gradio-container {
+  max-width: 1280px !important;
+  margin: 0 auto !important;
+  background: var(--avgen-bg) !important;
+  color: var(--avgen-ink) !important;
+}
+
+.app-header {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) auto;
+  gap: 24px;
+  align-items: end;
+  padding: 26px 0 12px;
+  border-bottom: 1px solid var(--avgen-line);
+}
+
+.app-header h1 {
+  margin: 0;
+  color: #111827 !important;
+  font-size: clamp(28px, 4vw, 44px);
+  font-weight: 850;
+  line-height: 1.05;
+  letter-spacing: 0;
+}
+
+.app-header p {
+  margin: 10px 0 0;
+  max-width: 860px;
+  color: #4b5563 !important;
+  font-size: 16px;
+}
+
+.header-links {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  justify-content: flex-end;
+}
+
+.header-links a {
+  color: var(--avgen-blue);
+  text-decoration: none;
+  border: 1px solid var(--avgen-line);
+  background: #fff;
+  padding: 8px 11px;
+  border-radius: 8px;
+  font-weight: 650;
+}
+
+.summary-grid {
+  display: grid;
+  grid-template-columns: repeat(5, minmax(130px, 1fr));
+  gap: 10px;
+  margin: 14px 0;
+}
+
+.summary-card,
+.method-card {
+  border: 1px solid var(--avgen-line);
+  background: var(--avgen-panel);
+  border-radius: 8px;
+  padding: 12px;
+}
+
+.summary-card span,
+.profile-total span,
+.eyebrow {
+  display: block;
+  color: #4f5d6e !important;
+  font-size: 12px;
+  text-transform: uppercase;
+  letter-spacing: 0;
+  font-weight: 700;
+}
+
+.summary-card strong {
+  display: block;
+  margin-top: 4px;
+  color: #111827 !important;
+  font-size: 26px;
+  font-weight: 850;
+  line-height: 1;
+}
+
+.summary-card small {
+  display: block;
+  margin-top: 6px;
+  color: #4b5563 !important;
+  min-height: 32px;
+}
+
+.table-shell {
+  overflow-x: auto;
+  border: 1px solid var(--avgen-line);
+  border-radius: 8px;
+  background: #fff;
+}
+
+.leaderboard-table {
+  width: 100%;
+  border-collapse: collapse;
+  min-width: 1180px;
+  font-size: 14px;
+}
+
+.leaderboard-table th,
+.leaderboard-table td {
+  padding: 11px 10px;
+  border-bottom: 1px solid #edf0f5;
+  vertical-align: middle;
+}
+
+.leaderboard-table tbody td {
+  color: var(--avgen-ink) !important;
+}
+
+.leaderboard-table thead th {
+  position: sticky;
+  top: 0;
+  background: #f1f4f8;
+  color: #202936 !important;
+  text-align: left;
+  white-space: nowrap;
+  z-index: 1;
+}
+
+.leaderboard-table thead th.sorted {
+  background: #e6eef8;
+  color: #111827 !important;
+  box-shadow: inset 0 -2px 0 var(--avgen-blue);
+}
+
+.sort-indicator {
+  display: inline-block;
+  margin-left: 6px;
+  color: var(--avgen-blue);
+  font-weight: 850;
+}
+
+.leaderboard-table tbody tr:hover {
+  background: #faf7f1;
+}
+
+.rank-cell {
+  width: 62px;
+  color: #4f5d6e !important;
+  font-weight: 700;
+}
+
+.model-cell {
+  min-width: 180px;
+  color: #16202d !important;
+  font-weight: 750;
+}
+
+.components-cell {
+  min-width: 220px;
+}
+
+.metric-cell {
+  text-align: right;
+  color: #253142 !important;
+  font-weight: 650;
+  font-variant-numeric: tabular-nums;
+}
+
+.metric-cell.best {
+  color: #0f6a43 !important;
+  font-weight: 800;
+  background: #edf8f2;
+}
+
+.metric-cell.second {
+  color: #8a4b08 !important;
+  font-weight: 750;
+  background: #fff7e8;
+}
+
+.component-badge,
+.type-badge {
+  display: inline-flex;
+  align-items: center;
+  max-width: 100%;
+  margin: 2px 4px 2px 0;
+  padding: 3px 8px;
+  border-radius: 999px;
+  font-size: 12px;
+  font-weight: 700;
+  white-space: nowrap;
+  border: 1px solid transparent;
+}
+
+.component-badge.proprietary,
+.type-badge.proprietary {
+  color: var(--avgen-orange);
+  background: #fff4e5;
+  border-color: #f2d4aa;
+}
+
+.component-badge.open,
+.type-badge.opensource {
+  color: var(--avgen-blue);
+  background: #edf5ff;
+  border-color: #c7dff8;
+}
+
+.component-badge.neutral,
+.type-badge.mixed {
+  color: var(--avgen-teal);
+  background: #eaf7f5;
+  border-color: #bde0dc;
+}
+
+.profile-panel {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) auto;
+  gap: 16px;
+  border: 1px solid var(--avgen-line);
+  background: #fff;
+  border-radius: 8px;
+  padding: 18px;
+}
+
+.profile-panel h2 {
+  margin: 2px 0 8px;
+  font-size: 26px;
+}
+
+.profile-total {
+  min-width: 140px;
+  text-align: right;
+}
+
+.profile-total strong {
+  display: block;
+  font-size: 42px;
+  line-height: 1;
+}
+
+.profile-grid {
+  grid-column: 1 / -1;
+  display: grid;
+  grid-template-columns: repeat(3, minmax(0, 1fr));
+  gap: 10px;
+}
+
+.profile-metric {
+  border: 1px solid #edf0f5;
+  border-radius: 8px;
+  padding: 10px;
+}
+
+.profile-metric-head {
+  display: flex;
+  justify-content: space-between;
+  gap: 10px;
+  font-variant-numeric: tabular-nums;
+}
+
+.bar-track {
+  height: 8px;
+  margin: 9px 0 6px;
+  border-radius: 999px;
+  background: #e7ebf1;
+  overflow: hidden;
+}
+
+.bar-fill {
+  height: 100%;
+  border-radius: inherit;
+  background: linear-gradient(90deg, var(--avgen-teal), var(--avgen-green));
+}
+
+.profile-metric small,
+.methodology p {
+  color: var(--avgen-muted);
+}
+
+.method-grid {
+  display: grid;
+  grid-template-columns: repeat(3, minmax(0, 1fr));
+  gap: 10px;
+  margin-bottom: 12px;
+}
+
+.method-card h3 {
+  margin: 0 0 4px;
+  font-size: 16px;
+}
+
+.method-card strong {
+  display: block;
+  font-size: 30px;
+  color: var(--avgen-blue);
+}
+
+.empty-state {
+  border: 1px solid var(--avgen-line);
+  background: #fff;
+  border-radius: 8px;
+  padding: 22px;
+  color: var(--avgen-muted);
+}
+
+.overview-image img {
+  border: 1px solid var(--avgen-line);
+  border-radius: 8px;
+  background: #fff;
+}
+
+.submission-copy {
+  color: var(--avgen-muted);
+  margin: 0 0 14px;
+}
+
+.submission-copy strong {
+  color: var(--avgen-ink);
+}
+
+.status-card {
+  border: 1px solid var(--avgen-line);
+  border-radius: 8px;
+  padding: 14px;
+  background: #fff;
+}
+
+.status-card strong {
+  display: block;
+  margin-bottom: 4px;
+}
+
+.status-card p {
+  margin: 4px 0 0;
+}
+
+.status-card.success {
+  border-color: #b8dbc9;
+  background: #f1faf5;
+}
+
+.status-card.error {
+  border-color: #efc2bd;
+  background: #fff3f1;
+}
+
+@media (max-width: 980px) {
+  .app-header {
+    grid-template-columns: 1fr;
+  }
+
+  .header-links {
+    justify-content: flex-start;
+  }
+
+  .summary-grid,
+  .profile-grid,
+  .method-grid {
+    grid-template-columns: 1fr 1fr;
+  }
+}
+
+@media (max-width: 640px) {
+  .summary-grid,
+  .profile-grid,
+  .method-grid {
+    grid-template-columns: 1fr;
+  }
+
+  .profile-panel {
+    grid-template-columns: 1fr;
+  }
+
+  .profile-total {
+    text-align: left;
+  }
+}
+"""
+
+
+HEADER = """
+<div class="app-header">
+  <div>
+    <h1>AVGen-Bench Leaderboard</h1>
+    <p>
+      A leaderboard for multi-granular evaluation of Text-to-Audio-Video generation,
+      covering visual/audio quality, synchronization, fine-grained controllability,
+      physical plausibility, and holistic semantic alignment.
+    </p>
+  </div>
+  <div class="header-links">
+    <a href="https://github.com/microsoft/AVGen-Bench" target="_blank" rel="noopener">GitHub</a>
+    <a href="https://arxiv.org/abs/2604.08540" target="_blank" rel="noopener">Paper</a>
+    <a href="https://huggingface.co/datasets/microsoft/AVGen-Bench" target="_blank" rel="noopener">Dataset</a>
+  </div>
+</div>
+"""
+
+
+with gr.Blocks(title="AVGen-Bench Leaderboard") as demo:
+    gr.HTML(HEADER)
+
+    with gr.Tab("Leaderboard"):
+        with gr.Row():
+            component_type = gr.Dropdown(
+                choices=["All", "Proprietary", "Open-source", "Mixed"],
+                value="All",
+                label="Component type",
+            )
+            sort_by = gr.Dropdown(
+                choices=SORT_CHOICES,
+                value="Total",
+                label="Sort item",
             )
+            sort_order = gr.Radio(
+                choices=["Descending", "Ascending", "Best first"],
+                value="Descending",
+                label="Sort order",
+            )
+            query = gr.Textbox(label="Search", placeholder="Model or component")
+
+        summary = gr.HTML()
+        table = gr.HTML()
+
+    with gr.Tab("Model Profile"):
+        model = gr.Dropdown(choices=model_choices(LEADERBOARD), value=model_choices(LEADERBOARD)[0], label="Model")
+        profile = gr.HTML()
 
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
+    with gr.Tab("Metric Scheme"):
+        gr.HTML(render_methodology())
+        if OVERVIEW_IMAGE.exists():
+            gr.Image(
+                value=str(OVERVIEW_IMAGE),
+                label="AVGen-Bench evaluation suite",
+                show_label=False,
+                interactive=False,
+                elem_classes=["overview-image"],
             )
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+    with gr.Tab("Submission"):
+        gr.HTML(
+            """
+            <p class="submission-copy">
+              Submit raw AVGen-Bench metrics for review. The app recomputes
+              <strong>Total</strong> from the raw metrics and sends the entry to a
+              pending-review backend. Accepted entries are still merged into the
+              official leaderboard manually.
+            </p>
+            """
+        )
+        with gr.Row():
+            submit_model = gr.Textbox(label="Model name", placeholder="Your Model")
+            submit_component_type = gr.Dropdown(
+                choices=["Proprietary", "Open-source", "Mixed"],
+                value="Open-source",
+                label="Component type",
+            )
+        submit_components = gr.Textbox(
+            label="Components",
+            placeholder="VideoModel (Open-source)|AudioModel (Open-source)",
+        )
+        with gr.Row():
+            submit_contact = gr.Textbox(label="Public contact", placeholder="GitHub handle or email")
+            submit_model_url = gr.Textbox(label="Model or paper URL", placeholder="https://...")
+            submit_results_url = gr.Textbox(label="Evaluation artifact URL", placeholder="https://...")
+        submit_notes = gr.Textbox(label="Notes", lines=3, placeholder="Optional evaluation details")
+
+        with gr.Accordion("Raw metric scores", open=True):
+            with gr.Row():
+                submit_vis = gr.Number(label="Vis")
+                submit_aud = gr.Number(label="Aud (PQ)")
+                submit_av = gr.Number(label="AV")
+                submit_lip = gr.Number(label="Lip")
+            with gr.Row():
+                submit_text = gr.Number(label="Text")
+                submit_face = gr.Number(label="Face")
+                submit_music = gr.Number(label="Music")
+                submit_speech = gr.Number(label="Speech")
+            with gr.Row():
+                submit_lophy = gr.Number(label="Lo-Phy")
+                submit_hiphy = gr.Number(label="Hi-Phy")
+                submit_holistic = gr.Number(label="Holistic")
+
+        submit_button = gr.Button("Submit for Review", variant="primary")
+        submit_status = gr.HTML()
+        submit_payload = gr.Code(label="Submission JSON", language="json")
+        gr.File(value=str(SUBMISSION_TEMPLATE), label="CSV template", interactive=False)
+
+    demo.load(
+        fn=update_leaderboard,
+        inputs=[component_type, query, sort_by, sort_order],
+        outputs=[summary, table],
+    )
+    demo.load(fn=update_profile, inputs=[model], outputs=[profile])
+
+    for control in [component_type, query, sort_by, sort_order]:
+        control.change(
+            fn=update_leaderboard,
+            inputs=[component_type, query, sort_by, sort_order],
+            outputs=[summary, table],
+        )
+
+    model.change(fn=update_profile, inputs=[model], outputs=[profile])
+    submit_button.click(
+        fn=submit_score,
+        inputs=[
+            submit_model,
+            submit_components,
+            submit_component_type,
+            submit_contact,
+            submit_model_url,
+            submit_results_url,
+            submit_notes,
+            submit_vis,
+            submit_aud,
+            submit_av,
+            submit_lip,
+            submit_text,
+            submit_face,
+            submit_music,
+            submit_speech,
+            submit_lophy,
+            submit_hiphy,
+            submit_holistic,
+        ],
+        outputs=[submit_status, submit_payload],
+        api_name="submit_score",
+    )
+
+
+if __name__ == "__main__":
+    launch_kwargs = {
+        "css": CSS,
+        "server_name": os.environ.get("GRADIO_SERVER_NAME", "0.0.0.0"),
+    }
+    port = os.environ.get("PORT") or os.environ.get("GRADIO_SERVER_PORT")
+    if port:
+        launch_kwargs["server_port"] = int(port)
+    demo.launch(**launch_kwargs)

data/leaderboard.csv

@@ -0,0 +1,14 @@
+Model,Components,Component Type,Vis,Aud (PQ),AV,Lip,Text,Face,Music,Speech,Lo-Phy,Hi-Phy,Holistic,Total,Source
+Seedance 2.0,Seedance 2.0 (Proprietary),Proprietary,0.945,7.15,0.15,4.14,74.83,60.95,28.12,94.09,3.89,83.16,89.61,72.07,AVGen-Bench README
+Veo 3.1-fast,Veo 3.1-fast (Proprietary),Proprietary,0.960,6.64,0.21,2.39,75.10,52.77,3.13,94.53,3.68,67.43,86.27,67.87,AVGen-Bench README
+Veo 3.1-quality,Veo 3.1-quality (Proprietary),Proprietary,0.954,6.77,0.24,3.59,76.53,52.90,5.00,96.09,3.74,68.53,84.10,66.28,AVGen-Bench README
+Sora-2,Sora-2 (Proprietary),Proprietary,0.848,5.91,0.25,4.50,74.84,51.17,7.81,88.63,4.05,78.95,88.89,64.16,AVGen-Bench README
+Wan2.6,Wan2.6 (Proprietary),Proprietary,0.959,7.15,0.30,4.32,76.95,49.27,1.75,89.33,3.69,66.92,80.98,62.97,AVGen-Bench README
+Seedance-1.5 Pro,Seedance-1.5 Pro (Proprietary),Proprietary,0.970,7.48,0.26,3.43,38.28,54.42,1.88,93.45,3.72,66.88,77.38,62.55,AVGen-Bench README
+Kling-V2.6,Kling-V2.6 (Proprietary),Proprietary,0.906,6.93,0.21,2.30,14.52,57.33,5.00,89.62,3.84,63.92,76.74,61.82,AVGen-Bench README
+LTX-2.3,LTX-2.3 (Open-source),Open-source,0.858,7.11,0.36,2.00,54.17,45.06,1.38,86.66,3.99,64.31,65.22,59.97,AVGen-Bench README
+NanoBanana2 + MOVA,NanoBanana2 (Proprietary)|MOVA (Open-source),Mixed,0.890,6.71,0.44,2.70,68.26,41.33,0.59,82.45,3.91,60.95,72.48,58.10,AVGen-Bench README
+LTX-2,LTX-2 (Open-source),Open-source,0.828,6.84,0.23,4.76,24.76,48.53,5.75,87.07,4.05,60.20,66.59,56.62,AVGen-Bench README
+Emu3.5 + MOVA,Emu3.5 (Open-source)|MOVA (Open-source),Open-source,0.911,6.80,0.38,4.83,64.72,48.44,0.62,81.74,3.89,55.85,66.55,56.12,AVGen-Bench README
+Wan2.2 + HunyuanVideo-Foley,Wan2.2 (Open-source)|HunyuanVideo-Foley (Open-source),Open-source,0.936,6.60,0.23,5.38,48.46,36.23,3.44,53.40,3.90,54.11,60.63,53.29,AVGen-Bench README
+Ovi,Ovi (Open-source),Open-source,0.839,6.31,0.37,5.40,41.36,49.05,11.25,76.49,3.93,52.92,57.45,52.02,AVGen-Bench README

data/submission_template.csv

@@ -0,0 +1,2 @@
+Model,Components,Component Type,Vis,Aud (PQ),AV,Lip,Text,Face,Music,Speech,Lo-Phy,Hi-Phy,Holistic,Total,Source
+Your Model,VideoModel (Open-source)|AudioModel (Open-source),Open-source,,,,,,,,,,,,,Your eval run or paper link

leaderboard.py

@@ -0,0 +1,384 @@
+from __future__ import annotations
+
+import html
+import math
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+import pandas as pd
+
+
+DATA_PATH = Path(__file__).parent / "data" / "leaderboard.csv"
+
+METRIC_COLUMNS = [
+    "Vis",
+    "Aud (PQ)",
+    "AV",
+    "Lip",
+    "Text",
+    "Face",
+    "Music",
+    "Speech",
+    "Lo-Phy",
+    "Hi-Phy",
+    "Holistic",
+    "Total",
+]
+
+SORT_COLUMNS = ["Rank", "Model", "Components", "Component Type", *METRIC_COLUMNS]
+SORT_CHOICES = [
+    ("Rank", "Rank"),
+    ("Model", "Model"),
+    ("Components", "Components"),
+    ("Type", "Component Type"),
+    *[(metric, metric) for metric in METRIC_COLUMNS],
+]
+
+LOWER_IS_BETTER = {"AV", "Lip"}
+
+NUMERIC_COLUMNS = METRIC_COLUMNS
+
+FORMATTERS = {
+    "Vis": "{:.3f}",
+    "Aud (PQ)": "{:.2f}",
+    "AV": "{:.2f}",
+    "Lip": "{:.2f}",
+    "Text": "{:.2f}",
+    "Face": "{:.2f}",
+    "Music": "{:.2f}",
+    "Speech": "{:.2f}",
+    "Lo-Phy": "{:.2f}",
+    "Hi-Phy": "{:.2f}",
+    "Holistic": "{:.2f}",
+    "Total": "{:.2f}",
+}
+
+GROUP_WEIGHTS = {
+    "Basic Uni-modal": 0.2,
+    "Basic Cross-modal": 0.2,
+    "Fine-grained": 0.6,
+}
+
+GROUP_DIMENSIONS = {
+    "Basic Uni-modal": ["Vis", "Aud (PQ)"],
+    "Basic Cross-modal": ["AV", "Lip"],
+    "Fine-grained": ["Text", "Face", "Music", "Speech", "Lo-Phy", "Hi-Phy", "Holistic"],
+}
+
+
+@dataclass(frozen=True)
+class Standing:
+    best: float | None
+    second: float | None
+
+
+def load_leaderboard(path: Path = DATA_PATH) -> pd.DataFrame:
+    df = pd.read_csv(path)
+    for column in NUMERIC_COLUMNS:
+        df[column] = pd.to_numeric(df[column], errors="coerce")
+    df = df.sort_values("Total", ascending=False, na_position="last").reset_index(drop=True)
+    df.insert(0, "Rank", range(1, len(df) + 1))
+    return df
+
+
+def filter_leaderboard(
+    df: pd.DataFrame,
+    component_type: str = "All",
+    query: str = "",
+    sort_by: str = "Total",
+    sort_order: str = "Descending",
+) -> pd.DataFrame:
+    view = df.copy()
+
+    if component_type != "All":
+        view = view[view["Component Type"] == component_type]
+
+    query = query.strip().lower()
+    if query:
+        mask = (
+            view["Model"].str.lower().str.contains(query, regex=False)
+            | view["Components"].str.lower().str.contains(query, regex=False)
+        )
+        view = view[mask]
+
+    sort_by = _normalize_sort_column(sort_by)
+    ascending = _is_ascending_sort(sort_by, sort_order)
+
+    if sort_by in view.columns:
+        sort_kwargs = {
+            "ascending": ascending,
+            "na_position": "last",
+            "kind": "mergesort",
+        }
+        if sort_by in {"Model", "Components", "Component Type"}:
+            sort_kwargs["key"] = lambda column: column.astype(str).str.casefold()
+        view = view.sort_values(sort_by, **sort_kwargs)
+
+    return view.reset_index(drop=True)
+
+
+def metric_standings(df: pd.DataFrame) -> dict[str, Standing]:
+    standings: dict[str, Standing] = {}
+    for metric in METRIC_COLUMNS:
+        values = sorted(
+            {float(v) for v in df[metric].dropna()},
+            reverse=metric not in LOWER_IS_BETTER,
+        )
+        standings[metric] = Standing(
+            best=values[0] if values else None,
+            second=values[1] if len(values) > 1 else None,
+        )
+    return standings
+
+
+def normalized_score(metric: str, value: float) -> float:
+    if metric == "Vis":
+        return _clamp(value * 100.0, 0.0, 100.0)
+    if metric == "Aud (PQ)":
+        return _clamp(value * 10.0, 0.0, 100.0)
+    if metric == "AV":
+        return _clamp(100.0 * (1.0 - value / 0.5), 0.0, 100.0)
+    if metric == "Lip":
+        return _clamp(100.0 * (1.0 - value / 8.0), 0.0, 100.0)
+    if metric == "Lo-Phy":
+        return _clamp(value * 20.0, 0.0, 100.0)
+    return _clamp(value, 0.0, 100.0)
+
+
+def compute_total_from_metrics(row: pd.Series) -> float:
+    group_scores: list[float] = []
+    group_weights: list[float] = []
+
+    for group_name, metrics in GROUP_DIMENSIONS.items():
+        values = []
+        for metric in metrics:
+            value = row.get(metric)
+            if pd.isna(value):
+                continue
+            values.append(normalized_score(metric, float(value)))
+        if values:
+            group_scores.append(sum(values) / len(values))
+            group_weights.append(GROUP_WEIGHTS[group_name])
+
+    if not group_scores:
+        return float("nan")
+
+    weighted = sum(score * weight for score, weight in zip(group_scores, group_weights))
+    return weighted / sum(group_weights)
+
+
+def render_summary(df: pd.DataFrame, view: pd.DataFrame) -> str:
+    top = df.sort_values("Total", ascending=False).iloc[0]
+    open_source = df[df["Component Type"] == "Open-source"].sort_values("Total", ascending=False)
+    best_open = open_source.iloc[0] if len(open_source) else None
+    best_av = df.sort_values("AV", ascending=True).iloc[0]
+    best_speech = df.sort_values("Speech", ascending=False).iloc[0]
+
+    cards = [
+        _summary_card("Models", f"{len(view)} / {len(df)}", "shown in current view"),
+        _summary_card("Top Total", _score(top["Total"]), str(top["Model"])),
+        _summary_card(
+            "Best Open-source",
+            _score(best_open["Total"]) if best_open is not None else "NA",
+            str(best_open["Model"]) if best_open is not None else "No entry",
+        ),
+        _summary_card("Lowest AV Offset", _score(best_av["AV"]), str(best_av["Model"])),
+        _summary_card("Highest Speech", _score(best_speech["Speech"]), str(best_speech["Model"])),
+    ]
+    return '<div class="summary-grid">' + "".join(cards) + "</div>"
+
+
+def render_table(
+    df: pd.DataFrame,
+    standings: dict[str, Standing],
+    sort_by: str = "Total",
+    sort_order: str = "Descending",
+) -> str:
+    if df.empty:
+        return '<div class="empty-state">No matching models.</div>'
+
+    sort_by = _normalize_sort_column(sort_by)
+    headers = [
+        ("Rank", "Rank"),
+        ("Model", "Model"),
+        ("Components", "Components"),
+        ("Type", "Component Type"),
+        *[(metric, metric) for metric in METRIC_COLUMNS],
+    ]
+    header_html = "".join(_header_cell(label, column, sort_by, sort_order) for label, column in headers)
+
+    rows = []
+    for _, row in df.iterrows():
+        cells = [
+            f'<td class="rank-cell">#{int(row["Rank"])}</td>',
+            f'<td class="model-cell">{html.escape(str(row["Model"]))}</td>',
+            f'<td class="components-cell">{render_component_badges(str(row["Components"]))}</td>',
+            f'<td>{_type_badge(str(row["Component Type"]))}</td>',
+        ]
+        for metric in METRIC_COLUMNS:
+            cells.append(_metric_cell(metric, row[metric], standings[metric]))
+        rows.append("<tr>" + "".join(cells) + "</tr>")
+
+    return (
+        '<div class="table-shell"><table class="leaderboard-table">'
+        f"<thead><tr>{header_html}</tr></thead><tbody>{''.join(rows)}</tbody>"
+        "</table></div>"
+    )
+
+
+def render_component_badges(components: str) -> str:
+    badges = []
+    for component in _split_components(components):
+        lowered = component.lower()
+        kind = "proprietary" if "proprietary" in lowered else "open" if "open-source" in lowered else "neutral"
+        label = component.replace(" (Proprietary)", "").replace(" (Open-source)", "")
+        badges.append(f'<span class="component-badge {kind}">{html.escape(label)}</span>')
+    return "".join(badges)
+
+
+def render_profile(df: pd.DataFrame, model: str) -> str:
+    if df.empty:
+        return ""
+    if not model or model not in set(df["Model"]):
+        model = str(df.sort_values("Total", ascending=False).iloc[0]["Model"])
+
+    row = df[df["Model"] == model].iloc[0]
+    metric_blocks = []
+    for metric in METRIC_COLUMNS[:-1]:
+        value = float(row[metric])
+        normalized = normalized_score(metric, value)
+        direction = "lower is better" if metric in LOWER_IS_BETTER else "higher is better"
+        metric_blocks.append(
+            f"""
+            <div class="profile-metric">
+              <div class="profile-metric-head">
+                <span>{html.escape(metric)}</span>
+                <strong>{_format_metric(metric, value)}</strong>
+              </div>
+              <div class="bar-track"><div class="bar-fill" style="width: {normalized:.1f}%"></div></div>
+              <small>{normalized:.1f} normalized, {direction}</small>
+            </div>
+            """
+        )
+
+    return f"""
+    <div class="profile-panel">
+      <div>
+        <p class="eyebrow">Model profile</p>
+        <h2>{html.escape(str(row["Model"]))}</h2>
+        <div class="profile-components">{render_component_badges(str(row["Components"]))}</div>
+      </div>
+      <div class="profile-total">
+        <span>Total</span>
+        <strong>{_score(row["Total"])}</strong>
+      </div>
+      <div class="profile-grid">{''.join(metric_blocks)}</div>
+    </div>
+    """
+
+
+def render_methodology() -> str:
+    groups = "".join(
+        f"""
+        <div class="method-card">
+          <h3>{html.escape(name)}</h3>
+          <strong>{weight:.1f}</strong>
+          <p>{html.escape(', '.join(metrics))}</p>
+        </div>
+        """
+        for name, weight in GROUP_WEIGHTS.items()
+        for metrics in [GROUP_DIMENSIONS[name]]
+    )
+    return f"""
+    <div class="methodology">
+      <div class="method-grid">{groups}</div>
+      <p>
+        Total uses AVGen-Bench Scheme 2: group-weighted normalized metrics with
+        Vis x 100, Aud(PQ) x 10, Lo-Phy x 20, AV = 100 * max(0, 1 - AV / 0.5),
+        Lip = 100 * max(0, 1 - Lip / 8), and the remaining metrics already on
+        a 0-100 scale.
+      </p>
+    </div>
+    """
+
+
+def model_choices(df: pd.DataFrame) -> list[str]:
+    return list(df.sort_values("Total", ascending=False)["Model"])
+
+
+def _metric_cell(metric: str, value: float, standing: Standing) -> str:
+    if pd.isna(value):
+        return '<td class="metric-cell muted">NA</td>'
+    numeric = float(value)
+    classes = ["metric-cell"]
+    if _close(numeric, standing.best):
+        classes.append("best")
+    elif _close(numeric, standing.second):
+        classes.append("second")
+    return f'<td class="{" ".join(classes)}">{_format_metric(metric, numeric)}</td>'
+
+
+def _type_badge(component_type: str) -> str:
+    kind = component_type.lower().replace("-", "").replace(" ", "")
+    return f'<span class="type-badge {html.escape(kind)}">{html.escape(component_type)}</span>'
+
+
+def _summary_card(label: str, value: str, detail: str) -> str:
+    return f"""
+    <div class="summary-card">
+      <span>{html.escape(label)}</span>
+      <strong>{html.escape(value)}</strong>
+      <small>{html.escape(detail)}</small>
+    </div>
+    """
+
+
+def _header_cell(label: str, column: str, sort_by: str, sort_order: str) -> str:
+    if column != sort_by:
+        return f"<th>{html.escape(label)}</th>"
+
+    direction = "ascending" if _is_ascending_sort(sort_by, sort_order) else "descending"
+    indicator = "&uarr;" if direction == "ascending" else "&darr;"
+    return (
+        f'<th class="sorted" aria-sort="{direction}">'
+        f"{html.escape(label)}"
+        f'<span class="sort-indicator" aria-hidden="true">{indicator}</span>'
+        "</th>"
+    )
+
+
+def _normalize_sort_column(sort_by: str) -> str:
+    if sort_by == "Type":
+        return "Component Type"
+    return sort_by if sort_by in SORT_COLUMNS else "Total"
+
+
+def _is_ascending_sort(sort_by: str, sort_order: str) -> bool:
+    if sort_order == "Best first":
+        return sort_by in LOWER_IS_BETTER
+    return sort_order == "Ascending"
+
+
+def _split_components(value: str) -> Iterable[str]:
+    return [part.strip() for part in value.split("|") if part.strip()]
+
+
+def _format_metric(metric: str, value: float) -> str:
+    return FORMATTERS[metric].format(float(value))
+
+
+def _score(value: float) -> str:
+    if pd.isna(value):
+        return "NA"
+    return f"{float(value):.2f}"
+
+
+def _close(left: float, right: float | None) -> bool:
+    if right is None:
+        return False
+    return math.isclose(float(left), float(right), rel_tol=0.0, abs_tol=1e-9)
+
+
+def _clamp(value: float, low: float, high: float) -> float:
+    return max(low, min(high, value))

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+pythonpath = .

requirements-dev.txt

@@ -0,0 +1 @@
+pytest

requirements.txt

@@ -1,16 +1,2 @@
-APScheduler
-black
-datasets
-gradio
-gradio[oauth]
-gradio_leaderboard==0.0.13
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
-numpy
-pandas
-python-dateutil
-tqdm
-transformers
-tokenizers>=0.15.0
-sentencepiece
+gradio>=6.0,<7
+pandas>=2.2,<4

submission.py

@@ -0,0 +1,311 @@
+from __future__ import annotations
+
+import html
+import json
+import math
+import os
+import re
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from leaderboard import compute_total_from_metrics
+
+
+SUBMISSION_METRICS = [
+    "Vis",
+    "Aud (PQ)",
+    "AV",
+    "Lip",
+    "Text",
+    "Face",
+    "Music",
+    "Speech",
+    "Lo-Phy",
+    "Hi-Phy",
+    "Holistic",
+]
+
+COMPONENT_TYPES = ["Proprietary", "Open-source", "Mixed"]
+
+METRIC_RANGES = {
+    "Vis": (0.0, 1.0),
+    "Aud (PQ)": (0.0, 10.0),
+    "AV": (0.0, 10.0),
+    "Lip": (0.0, 100.0),
+    "Text": (0.0, 100.0),
+    "Face": (0.0, 100.0),
+    "Music": (0.0, 100.0),
+    "Speech": (0.0, 100.0),
+    "Lo-Phy": (0.0, 5.0),
+    "Hi-Phy": (0.0, 100.0),
+    "Holistic": (0.0, 100.0),
+}
+
+
+class SubmissionError(ValueError):
+    pass
+
+
+def submit_score(
+    model: str,
+    components: str,
+    component_type: str,
+    contact: str,
+    model_url: str,
+    results_url: str,
+    notes: str,
+    vis: float,
+    aud_pq: float,
+    av: float,
+    lip: float,
+    text: float,
+    face: float,
+    music: float,
+    speech: float,
+    lo_phy: float,
+    hi_phy: float,
+    holistic: float,
+) -> tuple[str, str]:
+    try:
+        submission = build_submission(
+            model=model,
+            components=components,
+            component_type=component_type,
+            contact=contact,
+            model_url=model_url,
+            results_url=results_url,
+            notes=notes,
+            metrics={
+                "Vis": vis,
+                "Aud (PQ)": aud_pq,
+                "AV": av,
+                "Lip": lip,
+                "Text": text,
+                "Face": face,
+                "Music": music,
+                "Speech": speech,
+                "Lo-Phy": lo_phy,
+                "Hi-Phy": hi_phy,
+                "Holistic": holistic,
+            },
+        )
+        destination = persist_submission(submission)
+        status_html = render_submission_status(submission, destination)
+        return status_html, submission_to_json(submission)
+    except SubmissionError as exc:
+        return render_error_status(str(exc)), ""
+    except Exception as exc:
+        return render_error_status(f"Submission failed: {exc}"), ""
+
+
+def build_submission(
+    *,
+    model: str,
+    components: str,
+    component_type: str,
+    contact: str,
+    model_url: str,
+    results_url: str,
+    notes: str,
+    metrics: dict[str, Any],
+) -> dict[str, Any]:
+    model = _clean_required(model, "Model name", min_len=2, max_len=120)
+    components = _clean_required(components, "Components", min_len=2, max_len=240)
+    contact = _clean_required(contact, "Public contact", min_len=2, max_len=160)
+    model_url = _clean_required(model_url, "Model or paper URL", min_len=8, max_len=500)
+    results_url = _clean_required(results_url, "Evaluation artifact URL", min_len=8, max_len=500)
+    notes = (notes or "").strip()
+    if len(notes) > 2000:
+        raise SubmissionError("Notes must be 2000 characters or fewer.")
+    if component_type not in COMPONENT_TYPES:
+        raise SubmissionError("Component type must be Proprietary, Open-source, or Mixed.")
+    _validate_url(model_url, "Model or paper URL")
+    _validate_url(results_url, "Evaluation artifact URL")
+
+    metric_values = {metric: _validate_metric(metric, metrics.get(metric)) for metric in SUBMISSION_METRICS}
+    total = compute_total_from_metrics(pd.Series(metric_values))
+    if pd.isna(total):
+        raise SubmissionError("Could not compute Total from the submitted metrics.")
+
+    return {
+        "schema_version": 1,
+        "status": "pending_review",
+        "submitted_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        "model": model,
+        "components": components,
+        "component_type": component_type,
+        "contact": contact,
+        "model_url": model_url,
+        "evaluation_artifact_url": results_url,
+        "notes": notes,
+        "metrics": metric_values,
+        "computed_total": round(float(total), 4),
+    }
+
+
+def persist_submission(submission: dict[str, Any]) -> dict[str, str]:
+    backend = os.environ.get("SUBMISSION_BACKEND", "").strip().lower()
+    token = os.environ.get("GITHUB_TOKEN", "").strip()
+    repo = os.environ.get("GITHUB_REPO", "").strip()
+
+    if not backend:
+        backend = "github_issue" if token and repo else "local_file"
+
+    if backend == "disabled":
+        raise SubmissionError("Public submission is currently disabled.")
+    if backend == "github_issue":
+        if not token or not repo:
+            raise SubmissionError("GITHUB_TOKEN and GITHUB_REPO are required for github_issue backend.")
+        issue_url = create_github_issue(submission, repo, token)
+        return {"backend": "github_issue", "url": issue_url}
+    if backend == "local_file":
+        path = save_submission_file(submission)
+        return {"backend": "local_file", "path": str(path)}
+
+    raise SubmissionError(f"Unknown SUBMISSION_BACKEND: {backend}")
+
+
+def create_github_issue(submission: dict[str, Any], repo: str, token: str) -> str:
+    payload = {
+        "title": f"[Leaderboard Submission] {submission['model']}",
+        "body": github_issue_body(submission),
+        "labels": ["leaderboard-submission", "needs-review"],
+    }
+    request = urllib.request.Request(
+        url=f"https://api.github.com/repos/{repo}/issues",
+        data=json.dumps(payload).encode("utf-8"),
+        headers={
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "User-Agent": "AVGen-Bench-Leaderboard",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=20) as response:
+            data = json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as exc:
+        detail = exc.read().decode("utf-8", errors="replace")
+        raise SubmissionError(f"GitHub issue creation failed ({exc.code}): {detail}") from exc
+    except urllib.error.URLError as exc:
+        raise SubmissionError(f"Could not reach GitHub API: {exc.reason}") from exc
+
+    issue_url = data.get("html_url")
+    if not issue_url:
+        raise SubmissionError("GitHub API response did not include an issue URL.")
+    return str(issue_url)
+
+
+def save_submission_file(submission: dict[str, Any]) -> Path:
+    root = Path(os.environ.get("PENDING_SUBMISSION_DIR", "pending_submissions"))
+    root.mkdir(parents=True, exist_ok=True)
+    filename = f"{_slugify(submission['model'])}-{_compact_timestamp(submission['submitted_at_utc'])}.json"
+    path = root / filename
+    path.write_text(submission_to_json(submission) + "\n", encoding="utf-8")
+    return path
+
+
+def github_issue_body(submission: dict[str, Any]) -> str:
+    metrics = submission["metrics"]
+    metric_rows = "\n".join(f"| {metric} | {metrics[metric]} |" for metric in SUBMISSION_METRICS)
+    return f"""## Submission
+
+| Field | Value |
+|---|---|
+| Model | {submission['model']} |
+| Components | {submission['components']} |
+| Component Type | {submission['component_type']} |
+| Computed Total | {submission['computed_total']:.2f} |
+| Contact | {submission['contact']} |
+| Model or Paper URL | {submission['model_url']} |
+| Evaluation Artifact URL | {submission['evaluation_artifact_url']} |
+| Submitted At UTC | {submission['submitted_at_utc']} |
+
+## Raw Metrics
+
+| Metric | Value |
+|---|---:|
+{metric_rows}
+
+## Notes
+
+{submission['notes'] or 'None'}
+
+## Machine-Readable Payload
+
+```json
+{submission_to_json(submission)}
+```
+"""
+
+
+def submission_to_json(submission: dict[str, Any]) -> str:
+    return json.dumps(submission, ensure_ascii=False, indent=2, sort_keys=True)
+
+
+def render_submission_status(submission: dict[str, Any], destination: dict[str, str]) -> str:
+    if destination["backend"] == "github_issue":
+        detail = f'<a href="{html.escape(destination["url"])}" target="_blank" rel="noopener">GitHub issue</a>'
+    else:
+        detail = html.escape(destination["path"])
+    return f"""
+    <div class="status-card success">
+      <strong>Submission received for review.</strong>
+      <p>Total was recomputed from raw metrics: <b>{submission['computed_total']:.2f}</b>.</p>
+      <p>Destination: {detail}</p>
+    </div>
+    """
+
+
+def render_error_status(message: str) -> str:
+    return f"""
+    <div class="status-card error">
+      <strong>Submission not accepted.</strong>
+      <p>{html.escape(message)}</p>
+    </div>
+    """
+
+
+def _clean_required(value: str, label: str, *, min_len: int, max_len: int) -> str:
+    value = (value or "").strip()
+    if len(value) < min_len:
+        raise SubmissionError(f"{label} is required.")
+    if len(value) > max_len:
+        raise SubmissionError(f"{label} must be {max_len} characters or fewer.")
+    return value
+
+
+def _validate_url(value: str, label: str) -> None:
+    if not re.match(r"^https?://", value):
+        raise SubmissionError(f"{label} must start with http:// or https://.")
+
+
+def _validate_metric(metric: str, value: Any) -> float:
+    if value is None or value == "":
+        raise SubmissionError(f"{metric} is required.")
+    try:
+        numeric = float(value)
+    except (TypeError, ValueError) as exc:
+        raise SubmissionError(f"{metric} must be numeric.") from exc
+    if not math.isfinite(numeric):
+        raise SubmissionError(f"{metric} must be finite.")
+    low, high = METRIC_RANGES[metric]
+    if numeric < low or numeric > high:
+        raise SubmissionError(f"{metric} must be between {low:g} and {high:g}.")
+    return round(numeric, 6)
+
+
+def _slugify(value: str) -> str:
+    slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.strip().lower()).strip("-")
+    return slug or "submission"
+
+
+def _compact_timestamp(value: str) -> str:
+    return re.sub(r"[^0-9]", "", value)[:14]

tests/test_leaderboard.py

@@ -0,0 +1,80 @@
+import math
+
+from leaderboard import (
+    METRIC_COLUMNS,
+    filter_leaderboard,
+    load_leaderboard,
+    metric_standings,
+    render_component_badges,
+    render_table,
+)
+
+
+def test_loads_current_leaderboard_sorted_by_total():
+    df = load_leaderboard()
+
+    assert len(df) == 13
+    assert df.iloc[0]["Model"] == "Seedance 2.0"
+    assert df.iloc[0]["Rank"] == 1
+    assert df["Total"].is_monotonic_decreasing
+
+
+def test_filter_open_source_entries():
+    df = load_leaderboard()
+    view = filter_leaderboard(df, component_type="Open-source")
+
+    assert set(view["Component Type"]) == {"Open-source"}
+    assert "LTX-2.3" in set(view["Model"])
+    assert "Ovi" in set(view["Model"])
+
+
+def test_filter_sorts_every_metric_ascending_and_descending():
+    df = load_leaderboard()
+
+    for metric in METRIC_COLUMNS:
+        ascending = filter_leaderboard(df, sort_by=metric, sort_order="Ascending")
+        descending = filter_leaderboard(df, sort_by=metric, sort_order="Descending")
+
+        assert ascending[metric].is_monotonic_increasing
+        assert descending[metric].is_monotonic_decreasing
+
+
+def test_filter_sorts_non_metric_columns():
+    df = load_leaderboard()
+
+    by_model = filter_leaderboard(df, sort_by="Model", sort_order="Ascending")
+    by_type = filter_leaderboard(df, sort_by="Component Type", sort_order="Descending")
+
+    assert list(by_model["Model"]) == sorted(df["Model"], key=str.casefold)
+    assert list(by_type["Component Type"]) == sorted(
+        df["Component Type"],
+        key=str.casefold,
+        reverse=True,
+    )
+
+
+def test_render_table_marks_current_sort_column():
+    df = load_leaderboard()
+    standings = metric_standings(df)
+    html = render_table(df, standings, sort_by="Speech", sort_order="Ascending")
+
+    assert '<th class="sorted" aria-sort="ascending">Speech' in html
+    assert '<span class="sort-indicator" aria-hidden="true">&uarr;</span>' in html
+
+
+def test_metric_standings_handle_lower_is_better():
+    df = load_leaderboard()
+    standings = metric_standings(df)
+
+    assert math.isclose(standings["Total"].best, 72.07)
+    assert math.isclose(standings["AV"].best, 0.15)
+    assert math.isclose(standings["Lip"].best, 2.00)
+
+
+def test_component_badges_strip_type_suffixes():
+    html = render_component_badges("NanoBanana2 (Proprietary)|MOVA (Open-source)")
+
+    assert "NanoBanana2" in html
+    assert "MOVA" in html
+    assert "(Proprietary)" not in html
+    assert "(Open-source)" not in html

tests/test_submission.py

@@ -0,0 +1,67 @@
+import json
+import math
+
+import pytest
+
+from submission import SubmissionError, build_submission, save_submission_file
+
+
+VALID_METRICS = {
+    "Vis": 0.839,
+    "Aud (PQ)": 6.31,
+    "AV": 0.37,
+    "Lip": 5.40,
+    "Text": 41.36,
+    "Face": 49.05,
+    "Music": 11.25,
+    "Speech": 76.49,
+    "Lo-Phy": 3.93,
+    "Hi-Phy": 52.92,
+    "Holistic": 57.45,
+}
+
+
+def _valid_submission(**overrides):
+    data = {
+        "model": "Ovi",
+        "components": "Ovi (Open-source)",
+        "component_type": "Open-source",
+        "contact": "maintainer@example.com",
+        "model_url": "https://example.com/model",
+        "results_url": "https://example.com/results",
+        "notes": "test submission",
+        "metrics": VALID_METRICS,
+    }
+    data.update(overrides)
+    return build_submission(**data)
+
+
+def test_build_submission_recomputes_total():
+    submission = _valid_submission()
+
+    assert submission["status"] == "pending_review"
+    assert math.isclose(submission["computed_total"], 52.0174, abs_tol=1e-4)
+
+
+def test_build_submission_rejects_out_of_range_metric():
+    metrics = dict(VALID_METRICS)
+    metrics["Vis"] = 1.2
+
+    with pytest.raises(SubmissionError, match="Vis must be between 0 and 1"):
+        _valid_submission(metrics=metrics)
+
+
+def test_build_submission_requires_https_urls():
+    with pytest.raises(SubmissionError, match="Model or paper URL"):
+        _valid_submission(model_url="example.com/model")
+
+
+def test_save_submission_file(tmp_path, monkeypatch):
+    monkeypatch.setenv("PENDING_SUBMISSION_DIR", str(tmp_path))
+    submission = _valid_submission(model="My Model")
+
+    path = save_submission_file(submission)
+
+    assert path.exists()
+    assert path.parent == tmp_path
+    assert json.loads(path.read_text(encoding="utf-8"))["model"] == "My Model"