| from __future__ import annotations |
|
|
| import base64 |
| from pathlib import Path |
| from typing import List, Tuple |
|
|
| import gradio as gr |
| import pandas as pd |
|
|
| from src.data_loader import DataLoader |
| from src.leaderboard import Leaderboard |
| from src.plotter import Plotter |
| from src.radar_plotter import RadarPlotter |
| from src.styling import dataframe_to_html, get_academic_css |
|
|
| DATA_FILE = "./data/leaderboards.json" |
| APP_TITLE = "Urban Cup 2026 | EmbodiedCity Leaderboard" |
| REGISTRATION_FORM_URL = "https://rcnc7uacrkc1.feishu.cn/share/base/shrcns37gJlku9CVRaS9lXBZDcb" |
| SUBMISSION_UPLOAD_URL = "https://cloud.tsinghua.edu.cn/u/d/60c3058ed4ef4b8d90d4/" |
| EVENT_WEBSITE_URL = "https://fi.ee.tsinghua.edu.cn/RSUSHD2026/" |
| COMPETITION_GROUP_QR_PATH = Path(__file__).resolve().parent / "assets" / "competition_communication_group.png" |
| CONFERENCE_GROUP_QR_PATH = Path(__file__).resolve().parent / "assets" / "conference_official_group.jpg" |
|
|
|
|
| def build_title_markdown() -> str: |
| return f""" |
| # {APP_TITLE} |
| <span class="subtitle">Leaderboard-based evaluation for urban video question answering and goal-oriented embodied navigation</span> |
| """ |
|
|
|
|
| def build_registration_html() -> str: |
| return f""" |
| <div style="margin: 8px 0 26px 0; padding: 24px 26px; border-radius: 18px; background: linear-gradient(135deg, #f5fbff 0%, #ffffff 48%, #f7fff5 100%); border: 1px solid rgba(13, 120, 128, 0.16); box-shadow: 0 14px 34px rgba(13, 120, 128, 0.14);"> |
| <div style="display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 16px; margin-bottom: 16px;"> |
| <div> |
| <div style="font-size: 25px; font-weight: 900; color: #153b3f; line-height: 1.25;">Registration & Submission</div> |
| <div style="font-size: 16px; line-height: 1.65; color: #38565a; margin-top: 6px;"> |
| Register your team first, then upload a valid ZIP package. API submissions are preferred for faster organizer-side evaluation; model weights with inference code are also accepted. |
| </div> |
| </div> |
| <div style="display: flex; flex-wrap: wrap; gap: 10px;"> |
| <a href="{REGISTRATION_FORM_URL}" target="_blank" style="display: inline-block; padding: 11px 17px; border-radius: 999px; background: #0d7880; color: white; text-decoration: none; font-weight: 800; box-shadow: 0 8px 18px rgba(13, 120, 128, 0.22);">Open Registration Form</a> |
| <a href="{SUBMISSION_UPLOAD_URL}" target="_blank" style="display: inline-block; padding: 11px 17px; border-radius: 999px; background: #2f8f46; color: white; text-decoration: none; font-weight: 800; box-shadow: 0 8px 18px rgba(47, 143, 70, 0.20);">Upload ZIP Package</a> |
| </div> |
| </div> |
| <div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); gap: 14px; font-size: 14.5px; line-height: 1.65; color: #31464a;"> |
| <div style="padding: 14px 16px; border-radius: 14px; background: rgba(255, 255, 255, 0.82); border: 1px solid rgba(13, 120, 128, 0.10);"> |
| <strong style="color: #0d7880;">Tracks</strong><br> |
| Track 1: UrbanVideo-Bench for urban video QA.<br> |
| Track 2: EmbodiedNav-Bench for goal-oriented embodied navigation. |
| </div> |
| <div style="padding: 14px 16px; border-radius: 14px; background: rgba(255, 255, 255, 0.82); border: 1px solid rgba(13, 120, 128, 0.10);"> |
| <strong style="color: #0d7880;">ZIP Naming</strong><br> |
| Required format: <code>θ΅δΊδΈ_θ΅ιε _ε’ιεη§°_樑εεη§°_ζδΊ€η±»ε</code>. For updated versions, append <code>_V1</code>, <code>_V2</code>, etc. to the model name in the ZIP filename; repeated registration is not required. |
| </div> |
| <div style="padding: 14px 16px; border-radius: 14px; background: rgba(255, 255, 255, 0.82); border: 1px solid rgba(13, 120, 128, 0.10);"> |
| <strong style="color: #0d7880;">README Required</strong><br> |
| The ZIP must include a README with API endpoint/auth/calling instructions or model weights/inference code access, standard inference or evaluation entry, runtime requirements, and any rate-limit notes. |
| </div> |
| <div style="padding: 14px 16px; border-radius: 14px; background: rgba(255, 255, 255, 0.82); border: 1px solid rgba(13, 120, 128, 0.10);"> |
| <strong style="color: #0d7880;">Official Evaluation</strong><br> |
| Final leaderboard results are determined by organizer evaluation on a hidden test set. A registration without a valid uploaded ZIP package will be treated as invalid. |
| </div> |
| <div style="padding: 14px 16px; border-radius: 14px; background: rgba(255, 255, 255, 0.82); border: 1px solid rgba(13, 120, 128, 0.10);"> |
| <strong style="color: #0d7880;">Reference Baselines</strong><br> |
| Entries labeled <code>Reference Baseline</code> are not eligible for awards. Their reported results use the original benchmark test sets, which differ from the competition's final hidden test set, and are provided for reference only. |
| </div> |
| <div style="padding: 14px 16px; border-radius: 14px; background: rgba(255, 255, 255, 0.82); border: 1px solid rgba(13, 120, 128, 0.10);"> |
| <strong style="color: #0d7880;">Submission Limit</strong><br> |
| Each team may make up to three submissions per track. If more than three packages are submitted, only the three most recent submissions will be retained; newer versions will be evaluated first. |
| </div> |
| <div style="padding: 14px 16px; border-radius: 14px; background: rgba(255, 255, 255, 0.82); border: 1px solid rgba(13, 120, 128, 0.10);"> |
| <strong style="color: #0d7880;">API Deployment</strong><br> |
| If an internal GPU server (e.g., A800/H800) is not publicly reachable, use a pay-as-you-go VPS/ECS such as Alibaba Cloud or Volcengine as an HTTPS gateway, connected through WireGuard, FRP, or rathole. Keep inference and data processing on the backend; use a stable domain, TLS, sufficient video bandwidth, authentication, rate limits, and health checks. Keep the service available during evaluation, and never expose raw inference ports or model credentials. |
| </div> |
| </div> |
| <div style="margin-top: 14px; font-size: 14.5px; color: #5f6f72; line-height: 1.6;"> |
| Final submission deadline: <strong>July 30, 2026</strong>. Late submissions will not be evaluated. Leaderboard updates may be delayed by approximately one week. For version tracing, keep <code>θ΅ιε _ε’ιεη§°_樑εεη§°</code> consistent across registration and ZIP uploads. |
| </div> |
| </div> |
| """ |
|
|
|
|
| def build_image_data_uri(image_path: Path) -> str: |
| mime_type = "image/jpeg" if image_path.suffix.lower() in {".jpg", ".jpeg"} else "image/png" |
| encoded = base64.b64encode(image_path.read_bytes()).decode("ascii") |
| return f"data:{mime_type};base64,{encoded}" |
|
|
|
|
| def build_event_info_html() -> str: |
| competition_qr_data_uri = build_image_data_uri(COMPETITION_GROUP_QR_PATH) |
| conference_qr_data_uri = build_image_data_uri(CONFERENCE_GROUP_QR_PATH) |
| return f""" |
| <div style="margin: 28px 0 6px 0; padding: 24px 26px; border-radius: 18px; background: linear-gradient(135deg, #f5fbff 0%, #ffffff 48%, #f7fff5 100%); border: 1px solid rgba(13, 120, 128, 0.16); box-shadow: 0 14px 34px rgba(13, 120, 128, 0.14);"> |
| <div style="display: flex; flex-wrap: wrap; justify-content: space-between; gap: 24px; align-items: center;"> |
| <div style="flex: 1 1 320px;"> |
| <div style="font-size: 25px; font-weight: 900; color: #153b3f; line-height: 1.25; margin-bottom: 6px;"> |
| Event & Contact |
| </div> |
| <div style="font-size: 16px; line-height: 1.65; color: #38565a; margin-bottom: 8px; font-weight: 800;"> |
| Urban Cup 2026 Official Website |
| </div> |
| <div style="font-size: 14.5px; line-height: 1.7; color: #31464a; max-width: 760px;"> |
| For conference information, schedules, and related updates, please visit the official website. Scan the QR codes for the competition communication group and the conference official group. |
| </div> |
| <div style="margin-top: 18px; display: flex; flex-wrap: wrap; gap: 10px;"> |
| <a href="{EVENT_WEBSITE_URL}" target="_blank" style="display: inline-block; padding: 11px 18px; border-radius: 999px; background: #0d7880; color: #ffffff; text-decoration: none; font-weight: 900; box-shadow: 0 8px 18px rgba(13, 120, 128, 0.22);">Conference Website</a> |
| <span style="display: inline-block; padding: 11px 16px; border-radius: 999px; background: #2f8f46; color: #ffffff; font-weight: 850; box-shadow: 0 8px 18px rgba(47, 143, 70, 0.18);">Urban Cup 2026</span> |
| </div> |
| </div> |
| <div style="flex: 0 1 390px; display: flex; flex-wrap: wrap; justify-content: center; gap: 14px;"> |
| <div style="width: 176px; padding: 12px; border-radius: 18px; background: #ffffff; border: 1px solid rgba(13, 120, 128, 0.12); box-shadow: 0 12px 24px rgba(13, 120, 128, 0.16); text-align: center;"> |
| <img src="{competition_qr_data_uri}" alt="Competition communication group QR code" style="width: 152px; height: 152px; display: block; border-radius: 10px;" /> |
| <div style="margin-top: 8px; font-size: 12.5px; line-height: 1.35; color: #19484d; font-weight: 850;">Competition Communication Group</div> |
| </div> |
| <div style="width: 176px; padding: 12px; border-radius: 18px; background: #ffffff; border: 1px solid rgba(13, 120, 128, 0.12); box-shadow: 0 12px 24px rgba(13, 120, 128, 0.16); text-align: center;"> |
| <img src="{conference_qr_data_uri}" alt="Conference official group QR code" style="width: 152px; height: 152px; display: block; border-radius: 10px;" /> |
| <div style="margin-top: 8px; font-size: 12.5px; line-height: 1.35; color: #19484d; font-weight: 850;">Conference Official Group</div> |
| </div> |
| </div> |
| </div> |
| </div> |
| """ |
|
|
|
|
| def get_launch_kwargs() -> dict: |
| return { |
| "server_name": "0.0.0.0", |
| "server_port": 7860, |
| "prevent_thread_lock": False, |
| "ssr_mode": False, |
| } |
|
|
|
|
| def build_info_html(loader: DataLoader) -> str: |
| benchmark = loader.benchmark_config |
| if benchmark is None: |
| return "<div class='placeholder'>Benchmark information unavailable.</div>" |
|
|
| links_html = " ".join( |
| f"<a href='{link['url']}' target='_blank' style='color:#1976d2;text-decoration:none;font-weight:700;'>{link['label']}</a>" |
| for link in benchmark.get("links", []) |
| ) |
| dimensions = ", ".join(benchmark.get("dimensions", [])) |
| data_note = benchmark.get("dataNote", "") |
| data_note_html = f"<strong>Metric Notes:</strong> {data_note}<br>" if data_note else "" |
| award_note = benchmark.get("awardNote", "") |
| award_html = f"<strong>Awards:</strong> {award_note}<br>" if award_note else "" |
|
|
| return f""" |
| <div style="margin: 8px 0 20px 0; padding: 18px 22px; border-radius: 14px; background: linear-gradient(145deg, #ffffff, #f7fbff); box-shadow: 0 6px 22px rgba(41,128,185,0.10); border: 1px solid rgba(52,152,219,0.10);"> |
| <div style="font-size: 22px; font-weight: 800; color: #2c3e50; margin-bottom: 10px;"> |
| {benchmark['name']} |
| </div> |
| <div style="font-size: 16px; line-height: 1.7; color: #34495e; margin-bottom: 10px;"> |
| {benchmark['description']} |
| </div> |
| <div style="font-size: 15px; line-height: 1.7; color: #566573;"> |
| <strong>Scale:</strong> {benchmark['scale']}<br> |
| <strong>Dimensions:</strong> {dimensions}<br> |
| <strong>Evaluation:</strong> {benchmark['coverage']}<br> |
| {data_note_html} |
| {award_html} |
| <strong>Links:</strong> {links_html} |
| </div> |
| </div> |
| """ |
|
|
|
|
| def build_benchmark_tab(benchmark_id: str) -> Tuple: |
| loader = DataLoader(benchmark_id=benchmark_id, data_file=DATA_FILE) |
| loader.reload_data() |
| leaderboard = Leaderboard(loader) |
| plotter = Plotter(loader) |
| radar_plotter = RadarPlotter(loader) |
|
|
| benchmark = loader.benchmark_config |
| metric_choices = loader.get_metric_choices() |
| display_metric_choices = [loader.get_metric_label(metric) for metric in metric_choices] |
| default_internal_metric = benchmark["primaryMetric"] |
| default_display_metric = loader.get_metric_label(default_internal_metric) |
| default_selected_metrics = [ |
| loader.get_metric_label(metric) |
| for metric in benchmark.get("defaultMetrics", []) |
| if metric != default_internal_metric |
| ] |
|
|
| def to_internal_metric(metric_name: str) -> str: |
| return loader.to_internal_metric(metric_name) |
|
|
| def to_internal_metrics(metric_names: List[str]) -> List[str]: |
| return [to_internal_metric(metric_name) for metric_name in metric_names] |
|
|
| def build_radar_df(table_df: pd.DataFrame) -> pd.DataFrame: |
| displayed_models = table_df["Model"].tolist() if not table_df.empty else [] |
| return loader.get_dimension_dataframe(displayed_models) |
|
|
| def render_table( |
| metric_name: str, |
| top_k: int, |
| model_filter: str, |
| entry_type_filter: str, |
| sort_mode: str, |
| selected_metrics: List[str], |
| ) -> pd.DataFrame: |
| clean_metric = to_internal_metric(metric_name) |
| clean_selected_metrics = to_internal_metrics(selected_metrics) |
| return leaderboard.update_leaderboard( |
| metric=clean_metric, |
| top_k=top_k, |
| model_filter=model_filter, |
| entry_type_filter=entry_type_filter, |
| sort_mode=sort_mode, |
| selected_metrics=clean_selected_metrics, |
| ) |
|
|
| def reload_data(): |
| status_message = loader.reload_data() |
| table_df = render_table( |
| default_display_metric, |
| 20, |
| "", |
| "All", |
| "Auto", |
| default_selected_metrics, |
| ) |
| html_table = dataframe_to_html( |
| table_df, |
| column_label_map=loader.metric_display_map, |
| dimension_metrics=loader.dimension_metrics, |
| primary_metric=default_internal_metric, |
| ) |
| entry_type_update = gr.update(choices=loader.get_entry_type_choices(), value="All", interactive=True) |
| base_returns = ( |
| status_message, |
| entry_type_update, |
| build_info_html(loader), |
| html_table, |
| ) |
| if loader.dimension_metrics: |
| radar_df = build_radar_df(table_df) |
| radar_fig = radar_plotter.create_radar_chart(radar_df) |
| return base_returns + (radar_fig,) |
| return base_returns |
|
|
| def update_leaderboard_wrapper(metric, top_k, model_filter, entry_type_filter, sort_mode, selected_metrics): |
| table_df = render_table( |
| metric, |
| top_k, |
| model_filter, |
| entry_type_filter, |
| sort_mode, |
| selected_metrics, |
| ) |
| html_table = dataframe_to_html( |
| table_df, |
| column_label_map=loader.metric_display_map, |
| dimension_metrics=loader.dimension_metrics, |
| primary_metric=to_internal_metric(metric), |
| ) |
| if loader.dimension_metrics: |
| radar_df = build_radar_df(table_df) |
| radar_fig = radar_plotter.create_radar_chart(radar_df) |
| return html_table, radar_fig |
| return html_table |
|
|
| def create_comparison_plot_wrapper(model_filter, entry_type_filter, selected_plot_metric, plot_sort_mode): |
| internal_metric = to_internal_metric(selected_plot_metric) |
| return plotter.create_comparison_plot( |
| model_filter=model_filter, |
| entry_type_filter=entry_type_filter, |
| selected_plot_metric=internal_metric, |
| plot_sort_mode=plot_sort_mode, |
| display_metric_name=selected_plot_metric, |
| ) |
|
|
| status_box = gr.Markdown(f"Loading {benchmark['name']}...") |
| benchmark_info = gr.HTML(build_info_html(loader)) |
|
|
| with gr.Row(): |
| with gr.Column(scale=2): |
| metric_dropdown = gr.Dropdown( |
| label="Primary Ranking Metric", |
| choices=display_metric_choices, |
| value=default_display_metric, |
| interactive=True, |
| ) |
| with gr.Column(scale=1): |
| sort_mode_radio = gr.Radio( |
| label="Sort Order", |
| choices=["Auto", "Ascending (low β high)", "Descending (high β low)"], |
| value="Auto", |
| interactive=True, |
| ) |
| topk_slider = gr.Slider( |
| label="Display Top-K Models", |
| minimum=3, |
| maximum=max(20, len(loader.df_all) if loader.df_all is not None else 20), |
| value=min(20, len(loader.df_all) if loader.df_all is not None else 20), |
| step=1, |
| interactive=True, |
| ) |
|
|
| with gr.Row(): |
| metrics_select = gr.CheckboxGroup( |
| label="Additional Metrics to Display (π indicates summary dimensions)", |
| choices=[ |
| f"π {loader.get_metric_label(metric)}" if metric in loader.dimension_metrics else loader.get_metric_label(metric) |
| for metric in metric_choices |
| if metric != default_internal_metric |
| ], |
| value=[ |
| f"π {label}" if loader.to_internal_metric(label) in loader.dimension_metrics else label |
| for label in default_selected_metrics |
| ], |
| interactive=True, |
| ) |
|
|
| def normalize_selected_metrics(metric_names: List[str]) -> List[str]: |
| return [name.replace("π ", "") for name in metric_names] |
|
|
| with gr.Row(): |
| with gr.Column(scale=1): |
| model_filter_box = gr.Textbox( |
| label="Filter by Model Name", |
| placeholder="Enter model name (partial match)", |
| interactive=True, |
| ) |
| with gr.Column(scale=1): |
| entry_type_dropdown = gr.Dropdown( |
| label="Filter by Entry Type", |
| choices=loader.get_entry_type_choices(), |
| value="All", |
| interactive=True, |
| ) |
|
|
| with gr.Row(): |
| reload_button = gr.Button("π Reload Data", variant="secondary", size="sm") |
| update_button = gr.Button("β
Update Leaderboard", variant="primary", size="sm") |
|
|
| leaderboard_html = gr.HTML( |
| label="Leaderboard Table", |
| value="<div class='placeholder'>Leaderboard will be displayed here...</div>", |
| ) |
|
|
| radar_plot = None |
| if loader.dimension_metrics: |
| with gr.Row(): |
| radar_plot = gr.Plot(label="Dimension Radar Chart", format="png") |
|
|
| with gr.Row(): |
| with gr.Column(scale=2): |
| plot_metric_radio = gr.Radio( |
| label="Select Metric for Comparison Plot", |
| choices=display_metric_choices, |
| value=default_display_metric, |
| interactive=True, |
| ) |
| with gr.Column(scale=1): |
| plot_sort_radio = gr.Radio( |
| label="Plot Sort Order", |
| choices=["Ascending (low β high)", "Descending (high β low)"], |
| value="Descending (high β low)", |
| interactive=True, |
| ) |
| plot_update_button = gr.Button("π Generate Comparison Plot", variant="primary", size="sm") |
|
|
| comparison_plot = gr.Plot(label="Model Comparison Visualization", format="png") |
|
|
| reload_button.click( |
| fn=reload_data, |
| inputs=[], |
| outputs=[status_box, entry_type_dropdown, benchmark_info, leaderboard_html] |
| + ([radar_plot] if radar_plot is not None else []), |
| ) |
|
|
| update_button.click( |
| fn=lambda metric, top_k, model_filter, entry_type_filter, sort_mode, selected_metrics: update_leaderboard_wrapper( |
| metric, |
| top_k, |
| model_filter, |
| entry_type_filter, |
| sort_mode, |
| normalize_selected_metrics(selected_metrics), |
| ), |
| inputs=[ |
| metric_dropdown, |
| topk_slider, |
| model_filter_box, |
| entry_type_dropdown, |
| sort_mode_radio, |
| metrics_select, |
| ], |
| outputs=[leaderboard_html] + ([radar_plot] if radar_plot is not None else []), |
| ) |
|
|
| plot_update_button.click( |
| fn=create_comparison_plot_wrapper, |
| inputs=[ |
| model_filter_box, |
| entry_type_dropdown, |
| plot_metric_radio, |
| plot_sort_radio, |
| ], |
| outputs=[comparison_plot], |
| ) |
|
|
| return reload_data, [status_box, entry_type_dropdown, benchmark_info, leaderboard_html] + ( |
| [radar_plot] if radar_plot is not None else [] |
| ) |
|
|
|
|
| academic_css = get_academic_css() |
|
|
| with gr.Blocks(css=academic_css) as demo: |
| gr.Markdown(build_title_markdown(), elem_id="title") |
| gr.HTML( |
| """ |
| <div style="text-align: left; font-weight: 200; margin: 14px 0 18px 0; font-size: 19px; line-height: 1.7;"> |
| Explore reference results and evaluation resources for the two <strong style="font-size: 1.02em; color: #1976d2;">EmbodiedCity</strong> competition tracks: |
| <strong>UrbanVideo-Bench</strong> for urban video QA and <strong>EmbodiedNav-Bench</strong> for goal-oriented embodied navigation. |
| The competition follows a leaderboard format, while official rankings are produced separately through organizer evaluation on a hidden test set. |
| </div> |
| """ |
| ) |
| gr.HTML(build_registration_html()) |
|
|
| with gr.Tabs(): |
| with gr.Tab("UrbanVideo-Bench"): |
| urban_reload, urban_outputs = build_benchmark_tab("urbanvideo") |
|
|
| with gr.Tab("EmbodiedNav-Bench"): |
| nav_reload, nav_outputs = build_benchmark_tab("embodiednav") |
|
|
| gr.HTML(build_event_info_html()) |
|
|
| demo.load(fn=urban_reload, inputs=[], outputs=urban_outputs) |
| demo.load(fn=nav_reload, inputs=[], outputs=nav_outputs) |
|
|
|
|
| if __name__ == "__main__": |
| demo.launch(**get_launch_kwargs()) |
|
|
